From b1254756dfa131cbba205bfef3b2c5821e7e34b7 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Fri, 23 Oct 2026 16:38:32 +0900 Subject: [PATCH] benchmarks: Neo4j 2026.10.0, Memgraph 3.14.0 and Qdrant v1.20.0 comparator pins, with their clients Latest stable releases, pinned by amd64 manifest digest. Neo4j 2026.08.1 -> 2026.10.0 (c3903fa9...), Memgraph 3.13.1 -> 3.14.0 (c7478d17...), Qdrant v1.19.1 -> v1.20.0 (2d1b0bfd...). Clients: neo4j 6.3.1 -> 6.4.1, neo4j-rust-ext 6.3.1.0 -> 6.4.1.0, qdrant-client 1.19.1 -> 1.20.0. Vendor config unchanged (no release note requires a change). neo4j_memory_recommendation.json regenerated on the new image (DECISIONS #202): the recommendation is identical at all eight caps, so NEO4J_MEMORY is unchanged. The 12 stage drafts that bake the digests have a new sha list. Smoke on the laptop (graph, dense, cross-model): result digests equal across Neo4j, Memgraph and the composed stack; Neo4j quantization defaults re-probed (unset BINARY 3.0, NONE 1.0, SCALAR 1.5). Co-Authored-By: Claude Sonnet 5.5 --- benchmarks/experiments/COMPARATORS.md | 22 +++++++------- benchmarks/experiments/Dockerfile.composed | 8 ++--- benchmarks/experiments/PROTOCOL.md | 2 +- benchmarks/experiments/QUANTIZATION.md | 8 ++--- benchmarks/experiments/build_images.sh | 2 +- .../experiments/comparator_pins_check.py | 4 +-- benchmarks/experiments/e2_hybrid.py | 6 ++-- benchmarks/experiments/export_web.py | 5 +++- benchmarks/experiments/l3d_dense.py | 4 +-- .../neo4j_memory_recommendation.json | 4 +-- benchmarks/experiments/runner.py | 30 +++++++++---------- .../smoke_memgraph_server_limit.sh | 2 +- .../experiments/stage_drafts_2611.sha256 | 24 +++++++-------- .../experiments/test_gen_engines_doc.py | 4 +-- benchmarks/experiments/vendor_memory.py | 4 +-- 15 files changed, 66 insertions(+), 63 deletions(-) diff --git a/benchmarks/experiments/COMPARATORS.md b/benchmarks/experiments/COMPARATORS.md index 892c5489d3d..aaead891fb0 100644 --- a/benchmarks/experiments/COMPARATORS.md +++ b/benchmarks/experiments/COMPARATORS.md @@ -15,7 +15,7 @@ Which arms are on the October page, with their rows and columns, is PAGE-SPEC.md | DuckPGQ | `dbbench:duckdb` (duckdb==1.5.4) + `INSTALL duckpgq FROM community` at connect (extension version recorded on the row; `f386a6c` on 2026-09-17) | duckdb 1.5.4 + duckpgq | graph | embedded | Arrow INSERT SELECT into the persons and knows tables; a `CREATE PROPERTY GRAPH` over them answers every read in SQL/PGQ (`GRAPH_TABLE ... MATCH`), writes are plain SQL in one transaction (DECISIONS #103d, #92). Every graph question, interactive and analytics, LSQB's nine included, is expressible in SQL/PGQ and agrees with the other engines on the slice (DECISIONS #104b); the constructs are in `.notes/bench/COMPARATOR-DIALECTS.md`. | | SQLite | `dbbench:client` stdlib sqlite3 | 3.46.1 (the client image's Python) | documents (TPC), time series; lifecycle (`sqlite_lifecycle`, l5_lifecycle_sql.py; graph, the analytical view, and both vector situations declared with the engine's answer, sqlite-vec being its own arm) | embedded | executemany per transaction. Not at its defaults (DECISIONS #70; the PRAGMAs are in PROTOCOL.md section 7). | | LadybugDB | `ladybug==0.21.2` | 0.21.2 (from 2026-10-02, #129: 0.20.4 crashes the dense arm's mutation phase after a deleted vector key is re-inserted; 0.21.2 ran it clean, recall 0.998 after 1,000 deletes and re-inserts at micro) | graph; dense (#131, from the 26.10.1 measurement); cross-model (the official `vector` extension, downloaded at connect, from the 26.10.1 measurement, DECISIONS #131 item 4); lifecycle (`ladybug_lifecycle`, l5_lifecycle_embedded.py; ten indexes declared with the binder's answer that hash indexes go on primary keys only, the analytical view declared because PROJECT_GRAPH's projected graph is gone after a reopen, and sparse vectors declared because the vector index takes fixed-size arrays only; every node table keeps a primary-key hash index, which ArcadeDB's documents, graph, and time-series types do not carry, and the page says so) | embedded | COPY from CSV. 0.20.x added a physical-plan cache keyed on query text, and the lane re-executes each query text many times, so the jump from 0.19 was answer-checked: the lane's analytics shapes are byte-identical across 25 executions of one text, and ladybug#985 (a prepared query re-executed with other SKIP/LIMIT parameters returns the first page) does not reproduce and cannot be reached, since the lane passes no parameters. Dense (`ladybug_dense`): a FLOAT[DIM] node table loaded by COPY from an Arrow table, then the official `vector` extension (installed at connect) builds its two-layer HNSW at ml = 2 x the lane's M and mu = M, efc and efs at 100, metric l2. The cross-model arm's HNSW index is matched to the table (mu 16, ml 32, efc 100, efs 100; the engine defaults are 30, 60, and 200). Threads and buffer pool are fitted to the cell on every arm (PROTOCOL.md section 7). | -| Qdrant | `qdrant/qdrant@sha256:0699e773…` | v1.19.1 | dense, fp32 and int8; sparse, float32 and uint8 weights (`qdrant_sparse_uint8`, DECISIONS #135, from the 26.10.1 measurement); composed cross-model | served (the composed stack runs the same binary beside Neo4j in one container; its row is below) | upsert batches over REST on 6333 (`qdrant-client`'s default transport; no arm asks for gRPC). The sparse index's weight type is set in the definition (`float32`, the default, or `uint8`) and read back from the collection onto the row (`qdrant_sparse_datatype`). The sparse build ends when the collection's exact count equals the points sent and its status is green; until 2026-10-02 it ended at green, which on the laptop came with up to a third of the 100k slice still unapplied (`qdrant_points_at_first_green` records it). | +| Qdrant | `qdrant/qdrant@sha256:2d1b0bfd…` | v1.20.0 | dense, fp32 and int8; sparse, float32 and uint8 weights (`qdrant_sparse_uint8`, DECISIONS #135, from the 26.10.1 measurement); composed cross-model | served (the composed stack runs the same binary beside Neo4j in one container; its row is below) | upsert batches over REST on 6333 (`qdrant-client`'s default transport; no arm asks for gRPC). The sparse index's weight type is set in the definition (`float32`, the default, or `uint8`) and read back from the collection onto the row (`qdrant_sparse_datatype`). The sparse build ends when the collection's exact count equals the points sent and its status is green; until 2026-10-02 it ended at green, which on the laptop came with up to a third of the 100k slice still unapplied (`qdrant_points_at_first_green` records it). | | Milvus | `milvusdb/milvus@sha256:812cb91a…` + `quay.io/coreos/etcd@sha256:dc2bdc58…` + `milvusdb/minio@sha256:34c8e2f5…` (the vendor's Docker Compose standalone) | v3.0.3 | dense, sparse | served, three containers | BULK IMPORT, not insert batches: `RemoteBulkWriter` writes Parquet to Milvus's own MinIO and one import job loads it (file preparation, upload and job are all in ingest time, and are also recorded apart); then flush, a wait until every segment is Flushed and the index is Finished with nothing pending, the dense lane's compaction passes, load, and a check that the loaded segments hold every row. Each wait is timed on its own on the row. The timed search returns ids and distances only: no output fields, no vector field, no filter (recorded on the row). Deployment, changed 2026-10-22 after Zilliz's review (Jael Gu): v3.0.3 (2026-10-14), the latest 3.0 release, by amd64 manifest digest for all three images, as in the release's `milvus-standalone-docker-compose.yaml` (etcd v3.5.25, MinIO RELEASE.2024-12-18T13-15-44Z from the `milvusdb/minio` repository, which is pullable; the 3.0.1 compose's `minio/minio` tag was not). The runner starts the three containers itself in the cell's cpuset and network, and removes them after the cell. The three containers share the cell's one memory cap, as every other engine has it: etcd 512m and MinIO 1g are taken out of the cap and the Milvus container gets the rest, and the row's server-side memory, CPU, I/O and disk figures sum the three (the split is on the row); it does not call `docker compose`. The 2026-10-19 A/B (SIFT1M, ab.py) found no detectable search difference between the single container and Compose on 3.0.1; build times differ and are the point of the change. STORAGE MODE: Storage V2, Milvus's default, is the headline. Storage V3 is one switch, `BENCH_MILVUS_STORAGE=v3` (`common.storage.useLoonFFI=true`, https://milvus.io/docs/storage-v3.md), for a sensitivity run in its own results file; every row records `milvus_storage`. The dense arms set `dataCoord.segment.sealProportion` 0.12 -> 0.5 through the env var `DATACOORD_SEGMENT_SEALPROPORTION` (BUGS F8); the mounted copy of 3.0.1's `milvus.yaml` is gone, because a copy run under 3.0.3 would have kept 3.0.1's defaults (3.0.3's own file differs in about 60 lines, among them the proxy's `serverMaxRecvSize`). Every other setting is the compose default. The client is pymilvus 3.0.2 with the `bulk_writer` extra, the version Milvus's 3.0.3 release notes name. 3.0 rebuilt sparse retrieval around SINDI behind a version gate (selected only at vector index version 10 or higher); the harness sets no `targetVecIndexVersion`, `inverted_index_algo` or `quant_type`, so the sparse arm runs the stock algorithm and not a silently quantized one. HNSW M 16, efConstruction 100, ef 100 and the HNSW_SQ SQ8 arm are unchanged (Zilliz: a reasonable common starting point); equal ef is not equal recall, so every row carries recall@10 beside latency and the dense lane's beam sweep reports latency at matched recall (PROTOCOL.md, section 2). | | Elasticsearch | pinned in runner (`…@sha256:33178ff4…`) | server 9.5.4, client 9.5.1 | sparse; dense, fp32 and int8 (#131, from the 26.10.1 measurement) | served | bulk index, refresh, force-merge. The client line lags the server line: 9.5.1 is the newest `elasticsearch` release on PyPI while the server is at 9.5.4, so the two numbers differ by design and both are the latest of their own artifact. Dense (`elasticsearch_dense`, `elasticsearch_dense_int8`, 2026-10-02): the same server and heap; a `dense_vector` field at `index_options` hnsw (fp32) or int8_hnsw (its own default for floats this wide), m = the lane's M, ef_construction 100, L2, queried with `num_candidates` 100; the int8 arm re-scores its top k against the stored floats (`rescore_vector.oversample` 1.0), as the Qdrant int8 arm does; heap_max and the processor count read back from `_nodes`. | | Chroma, LanceDB, sqlite-vec | `dbbench:dense` (chromadb==1.5.9, lancedb==0.39.0, sqlite-vec==0.1.9) | as pinned | dense; lifecycle (DECISIONS #131 item 5, l5_lifecycle_embedded.py; each declares what it cannot build: a Chroma record needs an embedding and local Chroma has no sparse index, LanceDB's local connection runs no query language, and sqlite-vec's documents and time series are SQLite's own rows. LanceDB has no close on the connection, and a table's `close_lsm_writers()` only drains writers that `merge_insert` opens under an LSM write spec, which the arm's appends never do, so its session drops its handles; Chroma syncs at every write and LanceDB never does, neither with a setting, FAIRNESS F10) | embedded | add() / Arrow / executemany. LanceDB runs two arms from the 26.10.1 measurement (DECISIONS #135): `lancedb_dense`, `IVF_HNSW_SQ` (int8, as before), and `lancedb_dense_fp32`, `IVF_HNSW_FLAT` (unquantized), at the same degree, beam, `ef` and `nprobes`; each reads its index type back onto the row (`lancedb_index_type`, with the index details). | @@ -26,11 +26,11 @@ Which arms are on the October page, with their rows and columns, is PAGE-SPEC.md | MongoDB + MongoDB Search | built image `dbbench:mongo-search` (Dockerfile.mongosearch) FROM `mongo@sha256:41afd6e1…` + `mongodb/mongodb-community-search@sha256:24c0cd30…`; both digests pinned in the Dockerfile | mongod 8.2.12 + mongot-community 1.70.4 | dense vectors, cross-model | served, one container holding both processes | `$vectorSearch` over a `vectorSearch` index at the lane's matched point (PROTOCOL.md section 7). The vector field's `quantization` is `none` on `mongodb_dense` and `scalar` on `mongodb_dense_int8` (DECISIONS #135, from the 26.10.1 measurement), read back from mongot's index definition onto the row (`mongot_vector_quantization`). mongod runs WITH access control, unlike the document arm above (PROTOCOL.md section 7). Not `mongodb/mongodb-atlas-local`: MongoDB documents that image as "For evaluation only" and "not suitable for production use", and its vector-capable variant is a `:preview` tag. | | pgvector | `pgvector/pgvector@sha256:1d50c689…` (`0.8.6-pg18`) | PostgreSQL 18 + pgvector 0.8.6 | dense (`vector`, HNSW m=16, ef_construction=100, ef_search=100), sparse (`sparsevec`, HNSW, inner product) | served | COPY, then HNSW build. Memory and pools fitted to the cell (PROTOCOL.md section 7, FAIRNESS F6). | | TimescaleDB | `timescale/timescaledb@sha256:f7036933…` (`2.30.1-pg18`) | 2.30.1 on PostgreSQL 18 | time series (hypertable, `time_bucket`) | served | COPY into a hypertable, index on (host, ts). One of TSBS's home engines. Pinned at 2.30.1 and never 2.30.0: 2.30.0 introduced the `DeferredChunkAppend` planner node and 2.30.1 fixes that node producing DUPLICATE ROWS, which is an answer bug on this lane's own query shapes. Plain PostgreSQL runs beside it on the same image family (`postgres_ts`: COPY into a plain table, the same (host, ts DESC) index, `date_trunc` buckets), so the table shows what the extension buys. | -| Neo4j | `neo4j@sha256:e702d6b5…` (`2026.08.1-community`) | 2026.08.1 | graph, dense (its vector index, HNSW), cross-model | served | UNWIND batches over bolt. Dense queries use the Cypher 25 `SEARCH` clause; it has no candidate-count option and `LIMIT k` alone gave recall 0.50 at micro scale, so the adapter asks for `ef_search` (100) candidates and keeps the best k, the operating point every other engine runs at. Its vector index's quantization is set, never left to the default: 2026.08.1 builds BINARY (with a 3x search expansion) when the definition names none, which ours did through October (BUGS F164, DECISIONS #135); from the 26.10.1 measurement the fp32 arms set `NONE`, a `neo4j_dense_int8` arm sets `SCALAR`, and both read the applied type back onto the row (`neo4j_vector_quantization`). | +| Neo4j | `neo4j@sha256:c3903fa9…` (`2026.10.0-community`) | 2026.10.0 | graph, dense (its vector index, HNSW), cross-model | served | UNWIND batches over bolt. Dense queries use the Cypher 25 `SEARCH` clause; it has no candidate-count option and `LIMIT k` alone gave recall 0.50 at micro scale, so the adapter asks for `ef_search` (100) candidates and keeps the best k, the operating point every other engine runs at. Its vector index's quantization is set, never left to the default: 2026.08.1 to 2026.10.0 build BINARY (with a 3x search expansion) when the definition names none, which ours did through October (BUGS F164, DECISIONS #135); from the 26.10.1 measurement the fp32 arms set `NONE`, a `neo4j_dense_int8` arm sets `SCALAR`, and both read the applied type back onto the row (`neo4j_vector_quantization`). | | PostgreSQL + pgvector + Apache AGE | built image `dbbench:pg-age` (Dockerfile.pgage) FROM `pgvector/pgvector@sha256:1d50c689…` + `postgresql-18-age=1.8.0~rc0-2.pgdg12+1` from PGDG (the package pinned since 2026-10-02; unpinned, two builds could carry two AGE builds under one tag); both extension versions recorded on the row at connect | PostgreSQL 18 + pgvector 0.8.6 + AGE 1.8.0 | cross-model; graph, both workloads (DECISIONS #128, from the 26.10.1 measurement) | served | Cross-model: COPY plus Cypher CREATE through AGE; the strongest "one engine" rival to the page's central claim. Graph: the lane's own Cypher through `cypher()`, loaded by COPY into AGE's label tables with the graphids AGE's sequences assign, Message a PostgreSQL inheritance parent of Post and Comment, one btree on the Person `id` property (measured, #112). AGE's btree range scan from a `>=` bound skips the equal key (apache/age#2587), which costs the graph arm its two CRUD read-back digests, declared in `equivalence_check`. | -| Qdrant + Neo4j (the composed cross-model stack) | built image `dbbench:composed` (Dockerfile.composed) FROM `neo4j@sha256:e702d6b5…` with the Qdrant binary copied from `qdrant/qdrant@sha256:0699e773…`; both digests pinned in the Dockerfile (DECISIONS #117, BUGS F133) | Qdrant v1.19.1 + Neo4j 2026.08.1, stamped on the row as `qdrant:+neo4j:`, each read from its own server at connect | cross-model | served, one container holding both processes, so the one server cgroup caps, pins and measures both halves | Qdrant upsert batches over REST on 6333, then a payload index on `pid` and a wait for green status inside the index timer; Neo4j UNWIND batches over bolt. No transaction spans the two systems, which is what the arm is on the table to show. Qdrant runs with `RUN_MODE=production`, as its own image does (HNSW settings: PROTOCOL.md section 7). | +| Qdrant + Neo4j (the composed cross-model stack) | built image `dbbench:composed` (Dockerfile.composed) FROM `neo4j@sha256:c3903fa9…` with the Qdrant binary copied from `qdrant/qdrant@sha256:2d1b0bfd…`; both digests pinned in the Dockerfile (DECISIONS #117, BUGS F133) | Qdrant v1.20.0 + Neo4j 2026.10.0, stamped on the row as `qdrant:+neo4j:`, each read from its own server at connect | cross-model | served, one container holding both processes, so the one server cgroup caps, pins and measures both halves | Qdrant upsert batches over REST on 6333, then a payload index on `pid` and a wait for green status inside the index timer; Neo4j UNWIND batches over bolt. No transaction spans the two systems, which is what the arm is on the table to show. Qdrant runs with `RUN_MODE=production`, as its own image does (HNSW settings: PROTOCOL.md section 7). | | ArangoDB | `arangodb@sha256:4bc086d5…` (`arangodb:3.12.12`, also `latest` on 2026-10-06, from the 26.10.1 measurement; `arangodb@sha256:563cb2c0…`, `3.12.11`, before; the image reports "enterprise", which since 3.12 is the one self-managed build). License BSL 1.1 per the repository's LICENSE at tag v3.12.12 (the same file as at v3.12.11), so not open source; the page's license table says so | 3.12.12, RocksDB storage (3.12.11 until the 26.10.1 measurement) | documents (TPC), graph, dense, cross-model, time series (plain table) | served only: python-arango 8.3.5 is an HTTP client and the engine has no in-process mode | bulk import API in batches (`arangod --vector-index true`, PROTOCOL.md section 7). 3.12.11 had a defect: with 10,000 or more IVF lists every multithreaded query probes the same single list, so the October deep10m repetition answered at recall 0.0 after the calibration escalated to an exhaustive probe; 3.12.12 answers the same 12,643-list index at recall@10 0.9827 at the calibrated nProbe 167. The `3.12.11` tag was REBUILT on 2026-09-18 (its amd64 digest moved to `sha256:460b77c5349f…` while the pin held `sha256:563cb2c07af0…`). The calibration reads recall at nProbe = nLists first, on 20 held-out queries, and raises below a floor of 0.5 (a healthy exhaustive probe reads about 0.995). Its vector index is FAISS IVF, matched by effect rather than by knob, and the `ivf_*` fields on the row make the operating point auditable (FAIRNESS.md F7). From the 26.10.1 measurement an int8 arm, `arangodb_dense_int8` (DECISIONS #135), builds the same IVF through the FAISS factory string `IVF,SQ8`, so the lists hold 8-bit codes; its nProbe is calibrated to the same target, and the server's factory string is read back onto the row (`ivf_factory`, None on the fp32 arm). Time series (2026-09-15): no time-series type, so one collection with `ts` as epoch milliseconds and a persistent index over `["host", "ts"]`, SQLite's footing; buckets by `DATE_TRUNC(d.ts, 'minute')` / `DATE_TRUNC(d.ts, 'hour')`, which return ISO strings the digest reads as instants. | -| Memgraph | `memgraph/memgraph@sha256:4710bee1…` (`memgraph:3.13.1`, the latest self-hosted stable on 2026-09-17; `latest` resolves to the same digest). License BSL 1.1 for the engine (`memgraph/memgraph` LICENSE), with Memgraph's own Enterprise features gated by a license key that the arm never has; not open source | 3.13.1, IN_MEMORY_TRANSACTIONAL (the documented default: the graph in RAM, WAL and periodic snapshots on disk) | graph; dense (#131, from the 26.10.1 measurement); cross-model (its vector index, from the 26.10.1 measurement, DECISIONS #131 item 4: the index takes no HNSW parameters, so its degree and breadth stay at the engine's defaults, disclosed; candidates ranked by `vector_search.cosine_similarity`, which is L2 order on the lane's unit vectors) | served, Bolt through the same `neo4j==6.3.1` driver the Neo4j arm uses | UNWIND batches over bolt, the Neo4j arm's statements verbatim; every query on the lane runs unchanged and every digest matched Neo4j's at micro. Host-sized or unbounded defaults (Bolt workers, snapshot threads, `--memory-limit`, the query timeout, telemetry) are set from the cell and read back onto the row (PROTOCOL.md section 7, FAIRNESS F6). Durability from `SHOW CONFIG`: WAL fsynced every 100,000 transactions by default; the strict class sets 1 (both straced). Memory at SF10: about 1.1 KiB per stored object measured at 10k persons / 207k edges, so 2.3 GiB for the graph inside the tier's 24g. Dense (`memgraph_dense`, 2026-10-02): the same image and flags, plus `--storage-access-timeout-sec=172800` since #344 (PROTOCOL.md section 7); `CREATE VECTOR INDEX` (l2sq, f32) after an UNWIND load. Its HNSW parameters are the engine's and fixed: 3.13.1 builds every index from a default USearch 2.21.4 config (connectivity 16, doubled at the base, so hnswlib's M=16; expansion_add 128; expansion_search 64) and ignores any other config key, so ef_construction runs at 128 against the lane's 100 (disclosed on the row) and ef_search is matched by asking the index for 100 results and keeping 10, as the Neo4j arm does. An int8 arm, `memgraph_dense_int8` (DECISIONS #135, from the 26.10.1 measurement), sets `scalar_kind` `i8` in place of `f32`; both read the kind back from `SHOW VECTOR INDEX INFO` onto the row (`memgraph_vector_scalar_kind`). | +| Memgraph | `memgraph/memgraph@sha256:c7478d17…` (`memgraph:3.14.0`, the latest self-hosted stable on 2026-10-23; `latest` resolves to the same digest). License BSL 1.1 for the engine (`memgraph/memgraph` LICENSE), with Memgraph's own Enterprise features gated by a license key that the arm never has; not open source | 3.14.0, IN_MEMORY_TRANSACTIONAL (the documented default: the graph in RAM, WAL and periodic snapshots on disk) | graph; dense (#131, from the 26.10.1 measurement); cross-model (its vector index, from the 26.10.1 measurement, DECISIONS #131 item 4: the index takes no HNSW parameters, so its degree and breadth stay at the engine's defaults, disclosed; candidates ranked by `vector_search.cosine_similarity`, which is L2 order on the lane's unit vectors) | served, Bolt through the same `neo4j==6.4.1` driver the Neo4j arm uses | UNWIND batches over bolt, the Neo4j arm's statements verbatim; every query on the lane runs unchanged and every digest matched Neo4j's at micro. Host-sized or unbounded defaults (Bolt workers, snapshot threads, `--memory-limit`, the query timeout, telemetry) are set from the cell and read back onto the row (PROTOCOL.md section 7, FAIRNESS F6). Durability from `SHOW CONFIG`: WAL fsynced every 100,000 transactions by default; the strict class sets 1 (both straced). Memory at SF10: about 1.1 KiB per stored object measured at 10k persons / 207k edges, so 2.3 GiB for the graph inside the tier's 24g. Dense (`memgraph_dense`, 2026-10-02): the same image and flags, plus `--storage-access-timeout-sec=172800` since #344 (PROTOCOL.md section 7); `CREATE VECTOR INDEX` (l2sq, f32) after an UNWIND load. Its HNSW parameters are the engine's and fixed: 3.14.0 builds every index from a default USearch config (2.21.4 when read at 3.13.1; the 3.14.0 smoke read the same expansion_add 128 and expansion_search 64 back) (connectivity 16, doubled at the base, so hnswlib's M=16; expansion_add 128; expansion_search 64) and ignores any other config key, so ef_construction runs at 128 against the lane's 100 (disclosed on the row) and ef_search is matched by asking the index for 100 results and keeping 10, as the Neo4j arm does. An int8 arm, `memgraph_dense_int8` (DECISIONS #135, from the 26.10.1 measurement), sets `scalar_kind` `i8` in place of `f32`; both read the kind back from `SHOW VECTOR INDEX INFO` onto the row (`memgraph_vector_scalar_kind`). | | FalkorDB | `falkordb/falkordb@sha256:2756fdea…` (6.0.1, the amd64 digest, every arm from 2026-10-02, #129; `falkordb/falkordb@sha256:0a9fe4d1…`, `v4.20.6`, before). License SSPL 1.0 (`FalkorDB/FalkorDB` LICENSE.txt) on Redis 8.6.3 (RSALv2 / SSPLv1 / AGPLv3); not open source under the page's tests | 6.0.1 on Redis 8.10.2 (4.20.6 on Redis 8.6.3 until 2026-10-02) | graph; dense (#131, from the 26.10.1 measurement) | served, the Redis protocol through `falkordb==1.7.1` (`redis==8.1.0`) (2026-09-17) | UNWIND batches as `GRAPH.QUERY` calls, the Neo4j arm's statements verbatim (the client carries `$rows` as a CYPHER parameter prefix); every query on the lane runs unchanged and every digest matched Neo4j's at micro. The image's own `FALKORDB_ARGS` is replaced and `BROWSER=0` set (PROTOCOL.md section 7, FAIRNESS F6); 4.20.6 sized `THREAD_COUNT` from the host (16 threads under a 12-CPU cpuset), 6.0.1 follows the cpuset. Durability from `CONFIG GET`: RDB snapshots only by default (nothing synced at commit); the strict class runs AOF with `appendfsync always` through `REDIS_ARGS` (both straced). Dense (`falkordb_dense`, 2026-10-02) runs `falkordb/falkordb@sha256:2756fdea…` (6.0.1), because 4.20.6 and 4.22.0 crash on a write inside a vector statement (`repros/falkordb-vector-write/`); the graph arm runs the same digest. `vecf32` properties, `CREATE VECTOR INDEX` at M = the lane's M (hnswlib's), efConstruction and efRuntime 100, euclidean; the index builds in the background, so the build waits for `db.indexes()` to report OPERATIONAL (queried early, recall was 0.05). | The composed cross-model stack (Qdrant + Neo4j) is not an engine of its own: it is two of the rows above, at the same digests, wired together in one container (its row above). Its vector half was the Qdrant client's pure-Python in-memory local mode until 2026-09-26, so its rows up to then name `qdrant-local` and are marked `re-run` on the page until the 26.10.1 measurement's rows against the Qdrant server supersede them. @@ -52,18 +52,18 @@ Pins follow DECISIONS #87 (the latest self-hosted stable release at campaign tim | PostgreSQL + pgvector + AGE | PG18 + AGE 1.8.0 | built image, base as pgvector | | TimescaleDB | 2.30.1-pg18 | `sha256:f7036933154c…` | | MongoDB Search (mongot) | 1.70.4 | `sha256:24c0cd30baf0…` | -| Neo4j | 2026.08.1-community | `sha256:e702d6b535d9…` | -| Qdrant | v1.19.1 | `sha256:0699e7733a6f…` | +| Neo4j | 2026.10.0-community | `sha256:c3903fa96db3…` | +| Qdrant | v1.20.0 | `sha256:2d1b0bfd0a96…` | | Milvus | v3.0.3 | `sha256:812cb91ad04e…` (etcd v3.5.25 `sha256:dc2bdc588d2a…`, MinIO RELEASE.2024-12-18T13-15-44Z `sha256:34c8e2f52a59…`) | | Elasticsearch | 9.5.4 | `sha256:33178ff49e06…` | | QuestDB | 10.0.1 | `sha256:931af4156771…` | | ArangoDB | 3.12.12 | `sha256:4bc086d5050c…` | | LadybugDB | `ladybug==0.21.2` | client package | | LanceDB | `lancedb==0.39.0` | client package | -| Qdrant client | `qdrant-client==1.19.1` | client package | +| Qdrant client | `qdrant-client==1.20.0` | client package | | Elasticsearch client | `elasticsearch==9.5.1` | client package | -| Neo4j driver (Neo4j AND Memgraph arms) | `neo4j==6.3.1` | client package | -| Neo4j driver Rust extension (the vendor's documented accelerator, DECISIONS #173) | `neo4j-rust-ext==6.3.1.0` | client package | +| Neo4j driver (Neo4j AND Memgraph arms) | `neo4j==6.4.1` | client package | +| Neo4j driver Rust extension (the vendor's documented accelerator, DECISIONS #173) | `neo4j-rust-ext==6.4.1.0` | client package | | pgvector Python adapter (binary COPY, DECISIONS #174) | `pgvector==0.5.0` | client package | | ArcadeDB official Python driver (sync, httpx; the served ArcadeDB arms' headline client, CAMPAIGN section 7 row 72) | `arcadedb-driver==0.2.0` | client package | @@ -72,7 +72,7 @@ Pins follow DECISIONS #87 (the latest self-hosted stable release at campaign tim - **MongoDB 8.2.12**, not 8.3.11 (the one engine where #103d overrode #87): 8.3.11 refuses to start on the bench kernel (SERVER-121912), and an older 8.3.x that merely lacks the warning would run a configuration its vendor calls unsafe. Every MongoDB arm, the mongod half of the Search image included, holds at 8.2.12. - **DuckDB and DuckPGQ 1.5.4**, not 1.5.5: the community-extensions registry has no DuckPGQ build for 1.5.5 or 1.6.0 (DECISIONS #103d, #103e). - **SurrealDB server v3.2.4 and SDK 2.0.0**: the only higher numbers (`v3.3.0-beta.*`, `3.0.0b8`) are pre-releases. -- **Chroma 1.5.9, sqlite-vec 0.1.9, Memgraph 3.13.1, pymongo 4.18.1, python-arango 8.3.5, falkordb 1.7.1, redis 8.1.0**: the newest stable releases (sqlite-vec's `0.1.10a4` is a pre-release). **pymilvus 3.0.2** (with the `bulk_writer` extra, which brings `minio`, `azure-storage-blob` and `ml-dtypes`) is what Milvus's compatibility table names for a 3.0.3 server. +- **Chroma 1.5.9, sqlite-vec 0.1.9, Memgraph 3.14.0, pymongo 4.18.1, python-arango 8.3.5, falkordb 1.7.1, redis 8.1.0**: the newest stable releases (sqlite-vec's `0.1.10a4` is a pre-release). **pymilvus 3.0.2** (with the `bulk_writer` extra, which brings `minio`, `azure-storage-blob` and `ml-dtypes`) is what Milvus's compatibility table names for a 3.0.3 server. - **SQLite 3.46.1** is not a pin: it is the stdlib `sqlite3` of the `python:3.12-slim` base, read back at connect. **ArcadeDB is not a comparator**: it is pinned to our own wheel and the matching `arcadedb-c25` image (its row above). **Why some moves were smoked.** The PostgreSQL set moves as one major (#103d); AGE gated it, being the only member packaged per major (`postgresql-18-age`). PostgreSQL 18 turns data checksums on by default, a write-path cost the page keeps because it is PostgreSQL's own default. Neo4j is pinned at .1 because 2026.08.0 could fail queries through its resource monitoring, and a new index provider (`vector-2026.08`) is auto-selected for new indexes, so the dense operating point is a measurement and not an inheritance (the Neo4j row says how). @@ -102,7 +102,7 @@ Things an engine does that are not wrong and are not a performance property, but | Engine | Pin | Replaced by | |---|---|---| -| Neo4j 5-community (5.26.28) | `neo4j@sha256:4bae36af…` | 2026.07.1, then 2026.08.1 (`neo4j@sha256:e702d6b5…`) on every arm. | +| Neo4j 5-community (5.26.28) | `neo4j@sha256:4bae36af…` | 2026.07.1, then 2026.08.1 (`neo4j@sha256:e702d6b5…`), then 2026.10.0 (`neo4j@sha256:c3903fa9…`) on every arm. | | SurrealDB in memory (`mem://`) | `surrealdb==2.0.0` | the SDK's SurrealKV disk store | | ArcadeDB `417314c18` (a 26.10.1 snapshot, `arcadedb-c25:417314c18`) and September `26.9.1-dev · 8d6af9475` | wheel + `arcadedb-c25:` | `d36b4ca3a`, the 26.10.1 release's own jar commit (the ArcadeDB row above) | diff --git a/benchmarks/experiments/Dockerfile.composed b/benchmarks/experiments/Dockerfile.composed index 960e82cacbb..e4f015c2d79 100644 --- a/benchmarks/experiments/Dockerfile.composed +++ b/benchmarks/experiments/Dockerfile.composed @@ -1,5 +1,5 @@ -# The composed cross-model stack's server: Neo4j 2026.08.1-community + Qdrant -# v1.19.1 in one container (BUGS F133, DECISIONS #117). +# The composed cross-model stack's server: Neo4j 2026.10.0-community + Qdrant +# v1.20.0 in one container (BUGS F133, DECISIONS #117). # # WHY. The composed stack's vector half was QdrantClient(location=":memory:"), # the qdrant-client package's pure-Python local mode, which the comparator @@ -20,8 +20,8 @@ # are Debian 13), so nothing is fetched from a package mirror. # # Built as dbbench:composed by build_images.sh. -FROM qdrant/qdrant@sha256:0699e7733a6fa7fa7f6b95dcbed84ebb04584110da525cdfdef9f305c4f57738 AS qdrant -FROM neo4j@sha256:e702d6b535d9d3ae01ee7b132ec87aa40e23d3f0ace82fbfc344e2048cb81960 +FROM qdrant/qdrant@sha256:2d1b0bfd0a961275481ea09400fc98008216ae54e1b52ed7d24520d63050c644 AS qdrant +FROM neo4j@sha256:c3903fa96db31479c2a28e5d00e05260b039ff856f8b2cca7280c78299877731 COPY --from=qdrant /qdrant /qdrant COPY --from=qdrant /lib/x86_64-linux-gnu/libunwind-ptrace.so.0 /lib/x86_64-linux-gnu/libunwind-x86_64.so.8 /lib/x86_64-linux-gnu/libunwind.so.8 /lib/x86_64-linux-gnu/ COPY docker-conf/composed-entrypoint.sh /usr/local/bin/composed-entrypoint.sh diff --git a/benchmarks/experiments/PROTOCOL.md b/benchmarks/experiments/PROTOCOL.md index 6398c2c62bb..af8e4ea4644 100644 --- a/benchmarks/experiments/PROTOCOL.md +++ b/benchmarks/experiments/PROTOCOL.md @@ -222,7 +222,7 @@ An override nobody can find is a defect, whichever way it moves the number. Ever | ArcadeDB (documents, cross-model, graph message half) | `CREATE INDEX ... UNIQUE_HASH` on the ids a lane only looks up by equality: `Part(p_partkey)`, `OrderNew(okey)`, `Crud(ckey)`, `Product(pid)`, and the LDBC message half's vertex ids; `UNIQUE` (sorted) stays on `Person(id)` and wherever a key is ranged over or ordered | operating-point (the vendor's recommendation for the index kind, not a tuning knob) | ArcadeDB's maintainers' recommendation (ArcadeData/arcadedb#9169): equality-only ids take a hash index, keys also ranged or ordered stay sorted. The ingest and index columns carry the build cost and the transaction columns the gain. | FAIRNESS F14; `fairness_check.ARCADEDB_HASH_ID_INDEXES`, held equal to the DDL by `test_index_kinds.py`; row field `index_kinds`, the engine's own `schema:indexes` answer read after the schema is built, held to the registry by `fairness_check` F14d; a generated sentence built only from rows that carry the stamp | | Elasticsearch (L3s) | `xpack.security.enabled=false` | escape hatch | Harness plumbing; also removes TLS and auth from every ES latency measurement. | row field `es_security_enabled` (from the engine's `_xpack`), and a generated sentence; `override: es_security` | | Elasticsearch (L3s) | `number_of_replicas=0` | resource fitting | A single-node cluster cannot allocate a replica; the index would sit yellow forever. | row field `es_replicas` (from the index settings), and a generated sentence; `override: es_replicas` | -| composed Qdrant + Neo4j (E2) | the Qdrant server beside Neo4j in ONE container, `dbbench:composed` from `Dockerfile.composed` (the dense lane's pinned Qdrant v1.19.1 binary, Neo4j at the digest every Neo4j arm runs), reached over REST on 6333, with Qdrant's HNSW defaults (m=16, ef_construct=100) and a payload index on `pid` for the candidate filter (DECISIONS #117, BUGS F133) | resource fitting (one server cgroup for both halves) + operating-point (Qdrant's documented configuration for a filtered field) | Until 2026-09-26 the vector half was the client package's pure-Python local mode, rejected by the comparator review. Rows measured before carry `qdrant-local:` in `engine_version` and the page marks them `re-run`. | row field `engine_version` (`qdrant:+neo4j:`), COMPARATORS.md, page `re-run` note | +| composed Qdrant + Neo4j (E2) | the Qdrant server beside Neo4j in ONE container, `dbbench:composed` from `Dockerfile.composed` (the dense lane's pinned Qdrant v1.20.0 binary, Neo4j at the digest every Neo4j arm runs), reached over REST on 6333, with Qdrant's HNSW defaults (m=16, ef_construct=100) and a payload index on `pid` for the candidate filter (DECISIONS #117, BUGS F133) | resource fitting (one server cgroup for both halves) + operating-point (Qdrant's documented configuration for a filtered field) | Until 2026-09-26 the vector half was the client package's pure-Python local mode, rejected by the comparator review. Rows measured before carry `qdrant-local:` in `engine_version` and the page marks them `re-run`. | row field `engine_version` (`qdrant:+neo4j:`), COMPARATORS.md, page `re-run` note | | Neo4j (L2, L3d, E2) | the heap and page cache that `neo4j-admin server memory-recommendation --memory=` prints from the pinned image, one row per tier cap in `vendor_memory.NEO4J_MEMORY` (8g: heap 3600m, page cache 2g; 16g: 5g and 7g; 32g: 12g and 12g; 48g: 18g and 20g; the other caps in the same table), initial heap = max heap; was heap at half the cap and page cache = container memory less heap less 1 GiB, until DECISIONS #202 | vendor recommendation | The Neo4j manual: "Use the neo4j-admin server memory-recommendation command to get an initial recommendation", and "It is recommended to set these two parameters to the same value" for the initial and maximum heap. The tool is run for an empty data directory. The composed E2 container takes the same numbers; Qdrant lives in what they leave. The tool also prints `server.jvm.additional=-XX:+ExitOnOutOfMemoryError`, which the harness does not set. | row fields `server_pagecache` (the container's env) and `neo4j_pagecache` (the engine's `SHOW SETTINGS`), checked against each other by `fairness_check` F15, and a generated sentence; `override: neo4j_pagecache` | | Neo4j (graph) and the composed cross-model stack | `NEO4J_db_checkpoint_interval_time=5s` (default 15 minutes) | measurement (disk reading) | The store is checkpointed before the disk reading, which read 0.9 MB at SF1 under the default (BUGS F27). Preallocation stays at its default. | row fields `neo4j_checkpoint_interval` and `neo4j_checkpoint_interval_default` (the engine's `SHOW SETTINGS`), and a generated sentence; `override: neo4j_checkpoint` | | Memgraph (graph) | `--bolt-num-workers` and `--storage-snapshot-thread-count` from the cpuset, `--memory-limit` at 90% of the server cap, `--query-execution-timeout-sec=0`, `--telemetry-enabled=false` | resource fitting (workers, threads, memory) + escape hatch (timeout) | The three pools default to the host, and the 600 s default timeout would abort a whole-graph query that the lane's own budget is there to censor. | row fields `memgraph_bolt_workers`, `memgraph_snapshot_threads`, `memgraph_memory_limit_mib`; FAIRNESS F6; COMPARATORS.md | diff --git a/benchmarks/experiments/QUANTIZATION.md b/benchmarks/experiments/QUANTIZATION.md index 57ddb868afc..13e5bad9a79 100644 --- a/benchmarks/experiments/QUANTIZATION.md +++ b/benchmarks/experiments/QUANTIZATION.md @@ -11,19 +11,19 @@ The quantization survey of DECISIONS #131 item 8 (CAMPAIGN.md section 7 row 42), | engine (pin) | ships | our arm today | int8 arm | PQ arm (gated) | source | |---|---|---|---|---|---| | ArcadeDB (26.10.1 at the re-pin) | `LSM_VECTOR` quantization `NONE`, `INT8`, `BINARY`, `PRODUCT` | fp32 and INT8, embedded and served (no `PRODUCT` arm through 26.10.x: the SQL search scored candidates with the full-precision vectors, so PQ saved no memory there, [ArcadeData/arcadedb#10514](https://github.com/ArcadeData/arcadedb/issues/10514), fixed for 26.11.1 by [#10545](https://github.com/ArcadeData/arcadedb/pull/10545)) | exists | built, gated: `arcadedb_dense_embedded_pq`, `arcadedb_dense_server_pq`; `quantization: 'PRODUCT'`, `pqSubspaces` 32, `pqClusters` 256; SQL `vectorNeighbors` at the engine's default re-rank (10 candidates per neighbour on the stored vectors); the arm refuses an engine whose SQL rejects `rerank` | adapter (`l3d_dense._QUANT_DDL`, the engine's own error text for anything else) | -| Qdrant v1.19.1 | scalar (int8), product, binary, turbo | fp32 and scalar int8 (quantile 0.99, always_ram, rescore at search) | exists | built, gated: `qdrant_dense_pq`; `ProductQuantization` x16 (512 to 32 bytes, 32 subspaces of 256 centroids, `always_ram`), search with `rescore` and `oversampling` 10; the collection's compression is read back (`qdrant_quantization`) | image: `qdrant_client.models` in `dbbench:client` lists `ScalarQuantization` (`int8`), `ProductQuantization`, `BinaryQuantization`, `TurboQuantization` | +| Qdrant v1.20.0 | scalar (int8), product, binary, turbo (v1.20.0 adds the `turbo8` and `turbo16` datatypes; no arm uses them) | fp32 and scalar int8 (quantile 0.99, always_ram, rescore at search) | exists | built, gated: `qdrant_dense_pq`; `ProductQuantization` x16 (512 to 32 bytes, 32 subspaces of 256 centroids, `always_ram`), search with `rescore` and `oversampling` 10; the collection's compression is read back (`qdrant_quantization`) | image: `qdrant_client.models` in `dbbench:client` lists `ScalarQuantization` (`int8`), `ProductQuantization`, `BinaryQuantization`, `TurboQuantization` | | Milvus v3.0.1 | HNSW (fp32), `HNSW_SQ` (`sq_type` SQ8 and others), `HNSW_PQ`, `HNSW_PRQ`, IVF variants | HNSW fp32 and `HNSW_SQ` SQ8 | exists | built, gated: `milvus_dense_pq`; `HNSW_PQ` `m` 32, `nbits` 8, `refine` FP32, search `refine_k` 10; the index type and params are read back (`milvus_index_type`, `milvus_index_params`) | adapter (both arms build on the image); docs for the rest | | sqlite-vec 0.1.9 | `float32`, `int8`, `bit` column types, `vec_quantize_int8()` | fp32 and int8 | exists | none shipped (an exact scan) | image: `dbbench:dense` | | LanceDB 0.39.0 | `IVF_HNSW_FLAT` (unquantized HNSW in IVF partitions), `IVF_HNSW_SQ`, `IVF_HNSW_PQ`, `IVF_FLAT`, `IVF_SQ`, `IVF_PQ`, `IVF_RQ` | `lancedb_dense`: `IVF_HNSW_SQ`, INT8 (it kept its name, so its history stays one series); `lancedb_dense_fp32`: `IVF_HNSW_FLAT`, built 2026-10-02. The old premise (SQ as LanceDB's only HNSW offering) is gone from the adapter, runner, and paper-table comments | the SQ arm is the int8 arm; the fp32 arm now exists. Read back: `lancedb_index_type` (`index_stats`) and `lancedb_index_details` (`list_indices`: HNSW 16/100, `compression` sq 8 bits on the SQ arm only). Recall fp32 / int8: 0.9999 / 0.9902 at 5,000, 0.9994 / 0.9871 at 20,000 | no arm: `IVF_HNSW_PQ` trains its codebook per IVF partition, not over the corpus, so it is a different PQ setting; the fp32 and SQ arms stay its comparison | image: each type passed to `create_index` on a 3,000-row table in `dbbench:dense`; `HNSW_FLAT` and `HNSW_SQ` (without IVF) are refused; smoke: micro, tiny | | Chroma 1.5.9 | hnswlib, fp32 only | fp32 | none shipped | none shipped | adapter (only `hnsw:*` collection metadata exists); docs | | DuckDB VSS (duckdb 1.5.4) | HNSW over `FLOAT[]` arrays; no quantization setting | fp32 | none shipped | none shipped | image: `duckdb_settings()` in `dbbench:duckdb` lists only `hnsw_ef_search` and `hnsw_enable_experimental_persistence` | | pgvector 0.8.6 | `vector` (fp32), `halfvec` (fp16), `bit` (binary, by an expression index over `binary_quantize`), `sparsevec` | fp32 | no int8 type; fp16 and binary exist but are not int8, so no arm under #53 | none shipped (`halfvec` and `bit` are not PQ) | docs | -| Neo4j 2026.08.1 | `vector.quantization.type` `NONE`, `SCALAR`, `BINARY`; **unset means `BINARY`**, with `vector.default_search_expansion_factor` 3.0 | until 2026-10-02 **recorded fp32, ran BINARY**: `neo4j_dense` and `neo4j_e2` created their index without the key (BUGS F164). Now `neo4j_dense` and `neo4j_e2` set `NONE`, and both read the applied type back (`neo4j_vector_quantization`) | built: `neo4j_dense_int8` sets `SCALAR` (repin-prep `32c0a011da`). Each type brings its own `vector.default_search_expansion_factor` (NONE 1.0, SCALAR 1.5, BINARY 3.0 on 2026.08.1 and 2026.09.0, the candidate multiple searched before re-scoring); the lane matches search effort at ef 100 (Neo4j asks for 100 candidates and keeps 10), so both arms set 1.0 and read it back (`neo4j_vector_search_expansion`), as the Elasticsearch int8 arm sets the minimum oversample. 2026.09.0 (the re-pin candidate, `neo4j@sha256:eaab5879…`) keeps the BINARY default (`.notes` repros/neo4j-202609-quant) | none shipped (`NONE`, `SCALAR`, `BINARY`) | image: `SHOW VECTOR INDEXES` on an index created with exactly our options reports provider `vector-2026.08` and `vector.quantization.type: "BINARY"`; `INT8` and `FLOAT32` are syntax errors | +| Neo4j 2026.10.0 | `vector.quantization.type` `NONE`, `SCALAR`, `BINARY`; **unset means `BINARY`**, with `vector.default_search_expansion_factor` 3.0 | until 2026-10-02 **recorded fp32, ran BINARY**: `neo4j_dense` and `neo4j_e2` created their index without the key (BUGS F164). Now `neo4j_dense` and `neo4j_e2` set `NONE`, and both read the applied type back (`neo4j_vector_quantization`) | built: `neo4j_dense_int8` sets `SCALAR` (repin-prep `32c0a011da`). Each type brings its own `vector.default_search_expansion_factor` (NONE 1.0, SCALAR 1.5, BINARY 3.0 on 2026.08.1, 2026.09.0 and 2026.10.0 (the last re-probed 2026-10-23: unset gives BINARY, 3.0), the candidate multiple searched before re-scoring); the lane matches search effort at ef 100 (Neo4j asks for 100 candidates and keeps 10), so both arms set 1.0 and read it back (`neo4j_vector_search_expansion`), as the Elasticsearch int8 arm sets the minimum oversample. 2026.09.0 (the re-pin candidate, `neo4j@sha256:eaab5879…`) keeps the BINARY default (`.notes` repros/neo4j-202609-quant) | none shipped (`NONE`, `SCALAR`, `BINARY`) | image: `SHOW VECTOR INDEXES` on an index created with exactly our options reports provider `vector-2026.08` and `vector.quantization.type: "BINARY"`; `INT8` and `FLOAT32` are syntax errors | | MongoDB 8.2.12 + mongot-community 1.70.4 | `vectorSearch` index `quantization`: `none`, `scalar`, `binary` | `mongodb_dense`: `none` | built: `mongodb_dense_int8` sets `scalar`. Both read the field back from mongot's index definition (`mongot_vector_quantization`, `none` / `scalar`). Recall fp32 / int8: 0.9996 / 0.969 at 5,000, 0.9977 / 0.9699 at 20,000 | none shipped (`none`, `scalar`, `binary`) | adapter; smoke: micro, tiny (`scalar` accepted, built, queryable); docs for `binary` | | SurrealDB (core 2.3.10 embedded, 3.2.4 served) | HNSW `TYPE` F64, F32, I64, I32, I16: element types of the stored array, not quantization of float vectors | `TYPE F32` | none shipped | none shipped | adapter; docs | | ArangoDB 3.12.12 (3.12.11 until the 26.10.1 measurement) | FAISS IVF vector index; a custom FAISS `factory` string (SQ8, SQ4, PQ), forwarded to FAISS and validated at creation only from 3.12.12 | `arangodb_dense`: IVF flat, fp32 (`nLists`, `nProbe` calibrated) | built: `arangodb_dense_int8`, factory `IVF,SQ8` at the fp32 arm's nLists, nProbe calibrated to the same recall target; the server's factory string is read back (`ivf_factory`, None on the fp32 arm) beside its training state. Recall fp32 / int8, each calibrated on the held-out slice to the laptop's fallback target 0.95: 0.9402 / 0.9465 at 5,000, 0.9486 / 0.9447 at 20,000, at nProbe 17 / 19 and 23 / 24 | built, gated: `arangodb_dense_pq`; FAISS factory `IVF,PQ32x8` at the fp32 arm's nLists, nProbe calibrated to the same recall target; the factory string is read back (`ivf_factory`); the factory has no re-rank stage, so the arm answers from the codes | image: 20,000 SIFT vectors at nLists 566, both indexes `ready`, recall@10 at nProbe 1, 4, 16, 64: plain 0.345, 0.666, 0.932, 0.9985; SQ8 0.345, 0.6645, 0.928, 0.9905; docs (factory text, the 3.12.12 validation); smoke: micro, tiny; 3.12.12 (the pin since the 26.10.1 measurement; its digest is in COMPARATORS.md): `IVF70,SQ8` trains to ready and reads back unchanged at the same recall as 3.12.11, and an invalid factory string is now refused at creation, where 3.12.11 built an `unusable` index that answered by an exact scan (recall 1.0); the arm refuses any index that is not `ready` either way (`.notes` repros/arangodb-31212-sq8) | | Elasticsearch 9.5.4 (joins dense, row 37) | `hnsw`, `int8_hnsw`, `int4_hnsw`, `bbq_hnsw`, `bbq_disk`, `int8_flat` (all accepted); **default `int8_hnsw` at 96 and 128 dimensions** (m 16, ef_construction 100), `bbq_hnsw` with 3x oversampling from 384 | `elasticsearch_dense`: `hnsw`, set explicitly | built: `elasticsearch_dense_int8`, `int8_hnsw` with `rescore_vector.oversample` 1.0; both read the mapping's `index_options` back (`es_index_options`) | none shipped (`bbq` is binary) | image: dense_vector mappings on the pinned server, defaults read through `_mapping/field?include_defaults=true` | -| Memgraph 3.13.1 (joins dense, row 38) | `scalar_kind` f64, f32, f16, bf16, i8 (`b1` refused) | `memgraph_dense`: f32 | built: `memgraph_dense_int8`, `i8`; both read `scalar_kind` back from `SHOW VECTOR INDEX INFO` (`memgraph_vector_scalar_kind`). The i8 distances `vector_search.search` yields are integers, and the arm orders by them, so nothing re-ranks against the floats. Recall fp32 / int8: 0.9997 / 0.9423 at 5,000, 0.9992 / 0.9361 at 20,000 | none shipped | image: `CREATE VECTOR INDEX ... scalar_kind` on the pinned server, `SHOW VECTOR INDEX INFO`; smoke: micro, tiny | +| Memgraph 3.14.0 (joins dense, row 38) | `scalar_kind` f64, f32, f16, bf16, i8 (`b1` refused) | `memgraph_dense`: f32 | built: `memgraph_dense_int8`, `i8`; both read `scalar_kind` back from `SHOW VECTOR INDEX INFO` (`memgraph_vector_scalar_kind`). The i8 distances `vector_search.search` yields are integers, and the arm orders by them, so nothing re-ranks against the floats. Recall fp32 / int8: 0.9997 / 0.9423 at 5,000, 0.9992 / 0.9361 at 20,000 | none shipped | image: `CREATE VECTOR INDEX ... scalar_kind` on the pinned server, `SHOW VECTOR INDEX INFO`; smoke: micro, tiny | | FalkorDB (6.0.1 at the re-pin, row 38) | vector index over float32 | `falkordb_dense`: fp32 | none documented | none documented | docs | | LadybugDB 0.21.2 (row 38) | the official `vector` extension, HNSW over float arrays | `ladybug_dense`: fp32 | none documented | none documented | docs | @@ -42,7 +42,7 @@ Smoke, 2026-10-21, laptop, the first 5,000 siftsmall vectors and 80 timed querie | engine (pin) | ships | our arm today | int8 arm | source | |---|---|---|---|---| | ArcadeDB | `LSM_SPARSE_VECTOR` `weightQuantization` INT8 (the default), FP16, FP32 | INT8 (default) and FP32 | exists | adapter (`l3_sparse.ArcadeEmbedded.quant`, engine #5143) | -| Qdrant v1.19.1 | sparse `datatype` `float32` (default), `float16`, `uint8`, `turbo4` | `qdrant_sparse`: `float32`, now set in the definition | built: `qdrant_sparse_uint8`, `uint8`; both read the type back from the collection (`qdrant_sparse_datatype`). Recall fp32 / uint8: 1.0 / 1.0 on the 5,000-document synthetic corpus, 1.0 / 0.9952 on the 100,000-document Big-ANN SPLADE slice (once the arms' settle waits for every point, 2026-10-02: before that, both searched a collection still applying its last batches and read 0.93 and 0.88) | image: `qdrant_client.models.Datatype` in `dbbench:client`; smoke: synthetic micro, Big-ANN tiny | +| Qdrant v1.20.0 | sparse `datatype` `float32` (default), `float16`, `uint8`, `turbo4` | `qdrant_sparse`: `float32`, now set in the definition | built: `qdrant_sparse_uint8`, `uint8`; both read the type back from the collection (`qdrant_sparse_datatype`). Recall fp32 / uint8: 1.0 / 1.0 on the 5,000-document synthetic corpus, 1.0 / 0.9952 on the 100,000-document Big-ANN SPLADE slice (once the arms' settle waits for every point, 2026-10-02: before that, both searched a collection still applying its last batches and read 0.93 and 0.88) | image: `qdrant_client.models.Datatype` in `dbbench:client`; smoke: synthetic micro, Big-ANN tiny | | Milvus v3.0.1 | `SPARSE_FLOAT_VECTOR` with `SPARSE_INVERTED_INDEX` | float32 weights | none documented | adapter; docs | | Elasticsearch 9.5.4 | `sparse_vector`; weights kept at reduced precision by the engine, no setting | the default | none to choose | docs | | pgvector 0.8.6 | `sparsevec`, fp32 | fp32 | none shipped | docs | diff --git a/benchmarks/experiments/build_images.sh b/benchmarks/experiments/build_images.sh index ec0958d338f..4acfacfbab2 100755 --- a/benchmarks/experiments/build_images.sh +++ b/benchmarks/experiments/build_images.sh @@ -66,7 +66,7 @@ declare -A PKGS=( # arm's mutation phase after a deleted vector key is re-inserted, and 0.21.1 # carries #1074 (COPY linear for compressible columns) and #1075 (the pool # sized from the cgroup limit). 0.21.2 (2026-10-01) is the newest stable. - [client]="requests==2.34.2 psycopg[binary]==3.3.6 $HARNESS_PKGS surrealdb==2.0.0 qdrant-client==1.19.1 pymilvus[bulk_writer]==3.0.2 minio==7.2.20 azure-storage-blob==12.31.0 ml-dtypes==0.6.0 elasticsearch==9.5.1 neo4j==6.3.1 ladybug==0.21.2 pymongo==4.18.1 python-arango==8.3.5 falkordb==1.7.1 redis==8.1.0 neo4j-rust-ext==6.3.1.0 pgvector==0.5.0 arcadedb-driver==0.2.0" + [client]="requests==2.34.2 psycopg[binary]==3.3.6 $HARNESS_PKGS surrealdb==2.0.0 qdrant-client==1.20.0 pymilvus[bulk_writer]==3.0.2 minio==7.2.20 azure-storage-blob==12.31.0 ml-dtypes==0.6.0 elasticsearch==9.5.1 neo4j==6.4.1 ladybug==0.21.2 pymongo==4.18.1 python-arango==8.3.5 falkordb==1.7.1 redis==8.1.0 neo4j-rust-ext==6.4.1.0 pgvector==0.5.0 arcadedb-driver==0.2.0" [dense]="chromadb==1.5.9 lancedb==0.39.0 sqlite-vec==0.1.9 duckdb==1.5.4 $HARNESS_PKGS" ) # A GUARD, not a comment. The dev pin above survived because nothing checked diff --git a/benchmarks/experiments/comparator_pins_check.py b/benchmarks/experiments/comparator_pins_check.py index 459022c01b5..633fbb68977 100644 --- a/benchmarks/experiments/comparator_pins_check.py +++ b/benchmarks/experiments/comparator_pins_check.py @@ -84,7 +84,7 @@ def image_versions(): engine runs in a container takes its version from the image digest in `runner.BACKENDS`, not from a pypi pin, and comparing the two is a category error: `neo4j_graph` records the SERVER's 2026.07.1 while BACKEND_PKG maps - it to the `neo4j` python DRIVER pinned at 6.3.1, so a correct row was + it to the `neo4j` python DRIVER pinned at 6.4.1 (6.3.1 before 2026-10-23), so a correct row was reported STALE against a number that describes a different piece of software. Same for milvus (server 2.6.13 against pymilvus 3.0.1). @@ -158,7 +158,7 @@ def main(): # # Comparing it to a pypi pin is a category error and produced four # false STALEs: neo4j_graph's server 2026.07.1 against the `neo4j` - # DRIVER at 6.3.1, and milvus_dense/milvus_sparse's server 2.6.13 + # DRIVER at 6.4.1 (6.3.1 before 2026-10-23), and milvus_dense/milvus_sparse's server 2.6.13 # against `pymilvus` 3.0.1. Those rows are correct and pinned; the # check was comparing two different pieces of software. pkg, state = "image digest", "IMAGE-PINNED" diff --git a/benchmarks/experiments/e2_hybrid.py b/benchmarks/experiments/e2_hybrid.py index 89c62f35165..94015cceea0 100644 --- a/benchmarks/experiments/e2_hybrid.py +++ b/benchmarks/experiments/e2_hybrid.py @@ -1182,7 +1182,7 @@ def build(self, vecs, edges): s.run("UNWIND $rows AS r MATCH (a:Product {pid: r.s}), (b:Product {pid: r.d}) CREATE (a)-[:RELATED]->(b)", rows=eb[b0:b0 + BATCH]).consume() # `vector.quantization.type` set, never left to the default: Neo4j - # 2026.08.1 builds BINARY when the definition names none, which this + # 2026.08.1 to 2026.10.0 build BINARY when the definition names none (re-probed on 2026.10.0, 2026-10-23), which this # one did until 2026-10-02 (BUGS F164, DECISIONS #135). Read back. s.run(f"CREATE VECTOR INDEX prod_emb IF NOT EXISTS FOR (p:Product) ON (p.embedding) " f"OPTIONS {{indexConfig: {{`vector.dimensions`: {DIM}, `vector.similarity_function`: 'euclidean', " @@ -1194,7 +1194,7 @@ def build(self, vecs, edges): if cfg.get("vector.quantization.type") != "NONE": raise RuntimeError(f"neo4j vector index quantization read back " f"{cfg.get('vector.quantization.type')!r}, not 'NONE' (BUGS F164)") - # NONE's own default is 1.0 (2026.08.1, 2026.09.0); set and checked so + # NONE's own default is 1.0 (2026.08.1, 2026.09.0, 2026.10.0); set and checked so # a release that moves it cannot widen the candidate pool unseen. if float(cfg.get("vector.default_search_expansion_factor") or 0) != 1.0: raise RuntimeError(f"neo4j vector index search expansion read back " @@ -1266,7 +1266,7 @@ def close(self): class MemgraphE2: - """Memgraph 3.13.1 alone, served (DECISIONS #131 item 4, from the 26.10.1 + """Memgraph 3.13.1 alone (re-pinned to 3.14.0 on 2026-10-23), served (DECISIONS #131 item 4, from the 26.10.1 measurement): Product nodes carrying their embedding under Memgraph's vector index, RELATED edges, the views counter on the node; the hit, the hop, and the update run in one explicit Bolt transaction, and the injected diff --git a/benchmarks/experiments/export_web.py b/benchmarks/experiments/export_web.py index e4c27f5838d..a2525b054d2 100644 --- a/benchmarks/experiments/export_web.py +++ b/benchmarks/experiments/export_web.py @@ -690,6 +690,9 @@ def _image_version_names() -> dict[str, str]: # Read from each vendor's docs AND source at the version we run, never inferred # from the recall we measured: # +# Qdrant v1.20.0 fp32. (Re-checked 2026-10-23 at v1.20.0: DimWeight is still f32 and +# VectorStorageDatatype still #[default] Float32; the release notes list nothing +# sparse. The text below is the v1.19.1 reading, unchanged by that.) # Qdrant v1.19.1 fp32. RE-VERIFIED AT THE NEW PIN, and one # clause of the old citation was wrong. `datatype` is # NOT documented with a default: the schema text for @@ -805,7 +808,7 @@ def _image_version_names() -> dict[str, str]: # mongot's vectorSearch index with "quantization": "none", read back off # the created index and recorded on the row as mongot_quantization. "mongodb_dense": "fp32", - # Neo4j 2026.08.1 quantizes BINARY unless told otherwise (BUGS F164); the arms set NONE and SCALAR and + # Neo4j 2026.08.1 to 2026.10.0 quantize BINARY unless told otherwise (BUGS F164); the arms set NONE and SCALAR and # read the applied type back as neo4j_vector_quantization. "neo4j_dense": "fp32", "neo4j_dense_int8": "int8", "qdrant_dense": "fp32", diff --git a/benchmarks/experiments/l3d_dense.py b/benchmarks/experiments/l3d_dense.py index 6c7f8ea1027..dc452bb72a6 100644 --- a/benchmarks/experiments/l3d_dense.py +++ b/benchmarks/experiments/l3d_dense.py @@ -1396,7 +1396,7 @@ class Neo4jVector(Base): Neo4j exposes no per-query ef_search; the note under the table says so. THE QUANTIZATION IS SET, NOT LEFT TO THE DEFAULT (BUGS F164, DECISIONS - #135). Neo4j 2026.08.1 builds a vector index BINARY-quantized, with a + #135). Neo4j 2026.08.1 to 2026.10.0 build a vector index BINARY-quantized, with a search expansion factor of 3, when the definition names no `vector.quantization.type`, and this arm's definition named none until 2026-10-02 while it recorded fp32: every Neo4j dense row at the October @@ -1410,7 +1410,7 @@ class Neo4jVector(Base): QUANT_TYPE = "NONE" # THE SEARCH EXPANSION, SET AND READ BACK (2026-10-02). Each quantization # type brings its own default `vector.default_search_expansion_factor` - # (2026.08.1 and 2026.09.0: NONE 1.0, SCALAR 1.5, BINARY 3.0), the multiple + # (2026.08.1, 2026.09.0 and 2026.10.0: NONE 1.0, SCALAR 1.5, BINARY 3.0), the multiple # of the requested candidates the index searches before re-scoring. The lane # matches search effort at ef 100 and Neo4j, which has no per-query ef, asks # for 100 candidates and keeps 10, so a default 1.5 would give the int8 arm diff --git a/benchmarks/experiments/neo4j_memory_recommendation.json b/benchmarks/experiments/neo4j_memory_recommendation.json index 6435a0d22e0..a94694d9c41 100644 --- a/benchmarks/experiments/neo4j_memory_recommendation.json +++ b/benchmarks/experiments/neo4j_memory_recommendation.json @@ -1,6 +1,6 @@ { - "image": "neo4j@sha256:e702d6b535d9d3ae01ee7b132ec87aa40e23d3f0ace82fbfc344e2048cb81960", - "command": "docker run --rm -m 2g --entrypoint neo4j-admin neo4j@sha256:e702d6b535d9d3ae01ee7b132ec87aa40e23d3f0ace82fbfc344e2048cb81960 server memory-recommendation --memory={cap} --docker", + "image": "neo4j@sha256:c3903fa96db31479c2a28e5d00e05260b039ff856f8b2cca7280c78299877731", + "command": "docker run --rm -m 2g --entrypoint neo4j-admin neo4j@sha256:c3903fa96db31479c2a28e5d00e05260b039ff856f8b2cca7280c78299877731 server memory-recommendation --memory={cap} --docker", "caps": { "8g": { "heap_initial": "3600m", diff --git a/benchmarks/experiments/runner.py b/benchmarks/experiments/runner.py index 04d6ec045c9..c113a35f85f 100644 --- a/benchmarks/experiments/runner.py +++ b/benchmarks/experiments/runner.py @@ -830,7 +830,7 @@ def _milvus_arm(dense): "neo4j_graph": { "topology": "client_server", "image": "dbbench:client", - "server_image": "neo4j@sha256:e702d6b535d9d3ae01ee7b132ec87aa40e23d3f0ace82fbfc344e2048cb81960", # 2026.08.1-community + "server_image": "neo4j@sha256:c3903fa96db31479c2a28e5d00e05260b039ff856f8b2cca7280c78299877731", # 2026.10.0-community # heap parity with the ArcadeDB deployments (same per-scale heap) "server_env": ["-e", "NEO4J_AUTH=neo4j/dbbenchpass", "-e", "NEO4J_server_memory_heap_initial__size={heap}", @@ -870,7 +870,7 @@ def _milvus_arm(dense): "topology": "embedded", "image": "dbbench:client", }, - # MEMGRAPH 3.13.1 (2026-09-17), served, Bolt through the same neo4j driver + # MEMGRAPH 3.14.0 (2026-10-23; re-pinned from 3.13.1, first served 2026-09-17), served, Bolt through the same neo4j driver # the Neo4j arm uses and the lane's Cypher verbatim (l2_graph.MemgraphGraph). # Four of its defaults are sized from the HOST or unbounded, and each is # set from the cell here and read back onto the row by the adapter: @@ -894,7 +894,7 @@ def _milvus_arm(dense): "memgraph_graph": { "topology": "client_server", "image": "dbbench:client", - "server_image": "memgraph/memgraph@sha256:4710bee1ab5b47599876e30f17ae1679d0bbb2262d84dc06641521fecb7c89ce", # 3.13.1 + "server_image": "memgraph/memgraph@sha256:c7478d1737edb77fa2b1fcd4e37e34a68f10272d0ec7d7f9aa92d0afc8e0be9f", # 3.14.0 "server_cmd": ["--log-level=INFO", "--also-log-to-stderr=true", "--bolt-num-workers={ncpu}", "--storage-snapshot-thread-count={ncpu}", @@ -995,7 +995,7 @@ def _milvus_arm(dense): "memgraph_e2": { "topology": "client_server", "image": "dbbench:client", - "server_image": "memgraph/memgraph@sha256:4710bee1ab5b47599876e30f17ae1679d0bbb2262d84dc06641521fecb7c89ce", # 3.13.1 + "server_image": "memgraph/memgraph@sha256:c7478d1737edb77fa2b1fcd4e37e34a68f10272d0ec7d7f9aa92d0afc8e0be9f", # 3.14.0 "server_cmd": ["--log-level=INFO", "--also-log-to-stderr=true", "--bolt-num-workers={ncpu}", "--storage-snapshot-thread-count={ncpu}", @@ -1018,7 +1018,7 @@ def _milvus_arm(dense): "neo4j_e2": { "topology": "client_server", "image": "dbbench:client", - "server_image": "neo4j@sha256:e702d6b535d9d3ae01ee7b132ec87aa40e23d3f0ace82fbfc344e2048cb81960", # 2026.08.1-community + "server_image": "neo4j@sha256:c3903fa96db31479c2a28e5d00e05260b039ff856f8b2cca7280c78299877731", # 2026.10.0-community "server_env": ["-e", "NEO4J_AUTH=neo4j/dbbenchpass", "-e", "NEO4J_server_memory_heap_initial__size={heap}", "-e", "NEO4J_server_memory_heap_max__size={heap}", @@ -1198,8 +1198,8 @@ def _milvus_arm(dense): "topology": "client_server", "image": "dbbench:client", # NEO4J + QDRANT IN ONE CONTAINER since DECISIONS #117 (BUGS F133): the - # Neo4j digest every Neo4j arm runs (2026.08.1-community) with the - # Qdrant server binary from the dense lane's pinned v1.19.1 image, + # Neo4j digest every Neo4j arm runs (2026.10.0-community) with the + # Qdrant server binary from the dense lane's pinned v1.20.0 image, # built by build_images.sh from Dockerfile.composed, so the one server # cgroup caps, pins and measures both halves. The vector half was the # client's in-memory local mode until then. @@ -1300,7 +1300,7 @@ def _milvus_arm(dense): "qdrant_sparse": { "topology": "client_server", "image": "dbbench:client", - "server_image": "qdrant/qdrant@sha256:0699e7733a6fa7fa7f6b95dcbed84ebb04584110da525cdfdef9f305c4f57738", # v1.19.1 + "server_image": "qdrant/qdrant@sha256:2d1b0bfd0a961275481ea09400fc98008216ae54e1b52ed7d24520d63050c644", # v1.20.0 "server_port": 6333, "ready_regex": r"Qdrant (HTTP|gRPC) listening|Actix runtime found", }, @@ -1308,7 +1308,7 @@ def _milvus_arm(dense): "qdrant_sparse_uint8": { "topology": "client_server", "image": "dbbench:client", - "server_image": "qdrant/qdrant@sha256:0699e7733a6fa7fa7f6b95dcbed84ebb04584110da525cdfdef9f305c4f57738", # v1.19.1 + "server_image": "qdrant/qdrant@sha256:2d1b0bfd0a961275481ea09400fc98008216ae54e1b52ed7d24520d63050c644", # v1.20.0 "server_port": 6333, "ready_regex": r"Qdrant (HTTP|gRPC) listening|Actix runtime found", }, @@ -1414,7 +1414,7 @@ def _milvus_arm(dense): "qdrant_dense": { "topology": "client_server", "image": "dbbench:client", - "server_image": "qdrant/qdrant@sha256:0699e7733a6fa7fa7f6b95dcbed84ebb04584110da525cdfdef9f305c4f57738", # v1.19.1 + "server_image": "qdrant/qdrant@sha256:2d1b0bfd0a961275481ea09400fc98008216ae54e1b52ed7d24520d63050c644", # v1.20.0 "server_port": 6333, "ready_regex": r"Qdrant (HTTP|gRPC) listening|Actix runtime found", }, @@ -1425,7 +1425,7 @@ def _milvus_arm(dense): "qdrant_dense_int8": { "topology": "client_server", "image": "dbbench:client", - "server_image": "qdrant/qdrant@sha256:0699e7733a6fa7fa7f6b95dcbed84ebb04584110da525cdfdef9f305c4f57738", # v1.19.1 + "server_image": "qdrant/qdrant@sha256:2d1b0bfd0a961275481ea09400fc98008216ae54e1b52ed7d24520d63050c644", # v1.20.0 "server_port": 6333, "ready_regex": r"Qdrant (HTTP|gRPC) listening|Actix runtime found", }, @@ -1460,7 +1460,7 @@ def _milvus_arm(dense): "neo4j_dense": { "topology": "client_server", "image": "dbbench:client", - "server_image": "neo4j@sha256:e702d6b535d9d3ae01ee7b132ec87aa40e23d3f0ace82fbfc344e2048cb81960", # 2026.08.1-community + "server_image": "neo4j@sha256:c3903fa96db31479c2a28e5d00e05260b039ff856f8b2cca7280c78299877731", # 2026.10.0-community "server_env": ["-e", "NEO4J_AUTH=neo4j/dbbenchpass", "-e", "NEO4J_server_memory_heap_initial__size={heap}", "-e", "NEO4J_server_memory_heap_max__size={heap}", @@ -1471,7 +1471,7 @@ def _milvus_arm(dense): "neo4j_dense_int8": { "topology": "client_server", "image": "dbbench:client", - "server_image": "neo4j@sha256:e702d6b535d9d3ae01ee7b132ec87aa40e23d3f0ace82fbfc344e2048cb81960", # 2026.08.1-community + "server_image": "neo4j@sha256:c3903fa96db31479c2a28e5d00e05260b039ff856f8b2cca7280c78299877731", # 2026.10.0-community "server_env": ["-e", "NEO4J_AUTH=neo4j/dbbenchpass", "-e", "NEO4J_server_memory_heap_initial__size={heap}", "-e", "NEO4J_server_memory_heap_max__size={heap}", @@ -1511,7 +1511,7 @@ def _milvus_arm(dense): "memgraph_dense": { "topology": "client_server", "image": "dbbench:client", - "server_image": "memgraph/memgraph@sha256:4710bee1ab5b47599876e30f17ae1679d0bbb2262d84dc06641521fecb7c89ce", # 3.13.1, the graph arm's + "server_image": "memgraph/memgraph@sha256:c7478d1737edb77fa2b1fcd4e37e34a68f10272d0ec7d7f9aa92d0afc8e0be9f", # 3.14.0, the graph arm's "server_cmd": ["--log-level=INFO", "--also-log-to-stderr=true", "--bolt-num-workers={ncpu}", "--storage-snapshot-thread-count={ncpu}", @@ -1526,7 +1526,7 @@ def _milvus_arm(dense): "memgraph_dense_int8": { "topology": "client_server", "image": "dbbench:client", - "server_image": "memgraph/memgraph@sha256:4710bee1ab5b47599876e30f17ae1679d0bbb2262d84dc06641521fecb7c89ce", # 3.13.1, the graph arm's + "server_image": "memgraph/memgraph@sha256:c7478d1737edb77fa2b1fcd4e37e34a68f10272d0ec7d7f9aa92d0afc8e0be9f", # 3.14.0, the graph arm's "server_cmd": ["--log-level=INFO", "--also-log-to-stderr=true", "--bolt-num-workers={ncpu}", "--storage-snapshot-thread-count={ncpu}", diff --git a/benchmarks/experiments/smoke_memgraph_server_limit.sh b/benchmarks/experiments/smoke_memgraph_server_limit.sh index ff0253872d1..f2aa11ff9af 100755 --- a/benchmarks/experiments/smoke_memgraph_server_limit.sh +++ b/benchmarks/experiments/smoke_memgraph_server_limit.sh @@ -33,7 +33,7 @@ docker run -d --name "$NAME" --cpuset-cpus "$CORES" --memory "$MEM" -p "$PORT:76 for _ in $(seq 1 90); do docker logs "$NAME" 2>&1 | grep -q "Bolt server is fully armed and operational" && break; sleep 1; done docker logs "$NAME" 2>&1 | grep -q "Bolt server is fully armed and operational" || { echo "SMOKE FAIL: server not ready"; exit 1; } EXP=$PWD BENCH_SERVER_HOST=localhost BENCH_SERVER_PORT=$PORT LIMIT=$LIMIT SLACK=$SLACK \ - nice -n 19 taskset -c "$CORES" uv run --directory ../.. --with neo4j==6.3.1 python -I - <<'PY' || exit 1 + nice -n 19 taskset -c "$CORES" uv run --directory ../.. --with neo4j==6.4.1 python -I - <<'PY' || exit 1 import os, sys, time sys.path.insert(0, os.environ["EXP"]) import neo4j diff --git a/benchmarks/experiments/stage_drafts_2611.sha256 b/benchmarks/experiments/stage_drafts_2611.sha256 index d70d2b55c3a..bb1f930f07a 100644 --- a/benchmarks/experiments/stage_drafts_2611.sha256 +++ b/benchmarks/experiments/stage_drafts_2611.sha256 @@ -10,23 +10,23 @@ df750a505fd22aac47dc99e95654d7ced5f9b1d29307e11176f08e19a649497c arcadedb-relea ab56dde57ec91f0be4b9d33e2b4b456d00262e1000ffdc850dfa92618016394f arcadedb-release/qV09.sh ffa1ce8fcf59cdef2db6fdac12a7fb62026f344343f1474c8d63f3118a1ee794 arcadedb-release/qV10.sh 18dca8cff9b2f3e6deabf15609af9b80ae1d0e8eadc512959dbfa6ccbbbfc815 arcadedb-release/qV11.sh -97f73fdba88ca434bd17540bb69eeb14eb2bb220914ae82c9f20db756d31949b comparators-paper-5reps/qU01.sh -741d0a3518c3ba90666afa9495ac21148644423cb997c96a4fae5481753abcd8 comparators-paper-5reps/qU02.sh -a41a74e6d6ff0733b3e59f5c8a1efbb2160133639614a5d71f424410a7762dff comparators-paper-5reps/qU03.sh +5b76d6ce2ce321922c6a60fcaca6ef90a3259092b02733a1fb4397aede98ed11 comparators-paper-5reps/qU01.sh +33439f77e126128e6317e631d0e3c2e78a671bb35906822daa09aba1064c8f9d comparators-paper-5reps/qU02.sh +4bfeeba876b7b1ffc46c0f7b21e01446f9b4fbaae24b82809f6cf1ba79c4a022 comparators-paper-5reps/qU03.sh f28ff18b4a830df753e2087e0103e5b294f83d5ad370ca5c5e2f059bd7177727 comparators-paper-5reps/qU04.sh -b4b3b9514ade64c245a0e2ae199e635a4b4a5cece481cf4906704189a4630f3c comparators-paper-5reps/qU05.sh -0caad6f919a1a19db5ad04b5397608bdf8973e292e0be1adf65026a37d39c72b comparators-paper-5reps/qU06.sh +12fdfa972927a838d6edee20fb4113b9e340a940381724a5474f336fd3beb2ac comparators-paper-5reps/qU05.sh +c865c012d005f9b39eefe8728eca8401932f55d4ff1c0439680b3e352da7f7ec comparators-paper-5reps/qU06.sh 75d37fe01be94e4bac59eca71310f59bde5753e8d0988234d3d18fbae08a16eb comparators-paper-5reps/qU07.sh 724104505b46dba372beba53b626173ea6e3bf607171d97dcd707bbe3b140729 comparators-paper-5reps/qU08.sh -3bbaa1c5b2c2d319e917a4c8688d99b3a9531ee9c3e352eb1421e38b156ec151 comparators/qS01.sh -6d4b10f667c3c2dd5a42c04e47388b7468b842a13c4afd3b7393282eb07b2bbe comparators/qS02.sh +7a92545486d501ca7dede2854c9ad08e4fe0a57b9194c4e4a81f10478f31fea5 comparators/qS01.sh +2ed89731098262a461f88f554a1eda70b55a2356cac4f911f21ec2c82097eb63 comparators/qS02.sh 225d969b481a930a79097879df676eef1ae1385d3d3d78e4a8e8d3cca6048ba7 comparators/qS03.sh -2836bd3d69923b3fa52e38d5746244811752eb67c75441b43ae9a21fb964ac59 comparators/qS04.sh +dac1fad9daae58720a7db8695fe89b6d6e11a15a335adc0a9c07d59393337872 comparators/qS04.sh 7f5f6878f8e4e121d5c107e9cc5d0976f53d42b517190e471c97de18ed0a71a6 comparators/qS05.sh -3f13190d57232af92e181b62ad4a11d7967625c0af79db1d883e0004d40377dd comparators/qS06.sh -a73dae1fa5c57e808886e1dc19b44d090477e5eb4803d238137144f400057c19 comparators/qS07.sh +b954baa570f2382d369a9a93bacdc5a41332470e2658c7a2862e573f3235ab3d comparators/qS06.sh +cc496ce3d953a24ae11615fa7423e41534efc4a1232ced108dc673aeea819930 comparators/qS07.sh d86cb9af49ff51f26713f96d8346cbfc85673976378fa934c7f68e042c7797cb comparators/qS08.sh -a444bcb39221f9d38192f87288d0aad0279d48b568b32c6d7ad77cc72db70518 comparators/qS09.sh +56c5eea85dc6240511fba1c4f9fd5c49509081572d077640f53c2ca8970c11c2 comparators/qS09.sh 902f580f2ebdf2aee403697cd9e6909f0a4135df838cf214ff979079cadc86fd comparators/qS10.sh -d5fada14d1f3336fbf69a83c24932d7d9b487d298dc7f84665494acb3215301e comparators/qS11.sh +8af406c372873ab2bf0cf751c4b2eb7a4ae5ebefea9c92e3a2ff556325d8c51a comparators/qS11.sh f47edb43082f1c74888db94c06c98ed07aaa56ee667f4de54d513e43994e74a6 comparators/qS12.sh diff --git a/benchmarks/experiments/test_gen_engines_doc.py b/benchmarks/experiments/test_gen_engines_doc.py index 64fa940ce0d..0eff5fa02e7 100644 --- a/benchmarks/experiments/test_gen_engines_doc.py +++ b/benchmarks/experiments/test_gen_engines_doc.py @@ -47,8 +47,8 @@ def neo4j_rows(): for scale, cache in (("sf1", "3.00GiB"), ("sf10", "11.00GiB")): for cls, text in (("relaxed", "fsync at commit"), ("strict", "fsync at commit")): out.append(row(run_id=f"n_{scale}_{cls}", backend="neo4j_graph", scale=scale, durability_class=cls, - durability=text, engine_version="neo4j:2026.08.1", - server_image_ref="neo4j@sha256:e702d6b535d9", + durability=text, engine_version="neo4j:2026.10.0", + server_image_ref="neo4j@sha256:c3903fa96db3", neo4j_pagecache=cache, neo4j_checkpoint_interval="5s", neo4j_checkpoint_interval_default="15m", server_pagecache=cache.replace("GiB", "g"))) return out diff --git a/benchmarks/experiments/vendor_memory.py b/benchmarks/experiments/vendor_memory.py index 669a43459a1..6e1a08f5f84 100644 --- a/benchmarks/experiments/vendor_memory.py +++ b/benchmarks/experiments/vendor_memory.py @@ -20,8 +20,8 @@ HERE = os.path.dirname(os.path.abspath(__file__)) NEO4J_RECOMMENDATION_FILE = os.path.join(HERE, "neo4j_memory_recommendation.json") -# The image the tool ran from: the digest every Neo4j arm in runner.BACKENDS pins (2026.08.1-community). -NEO4J_IMAGE_DIGEST = "sha256:e702d6b535d9d3ae01ee7b132ec87aa40e23d3f0ace82fbfc344e2048cb81960" +# The image the tool ran from: the digest every Neo4j arm in runner.BACKENDS pins (2026.10.0-community). +NEO4J_IMAGE_DIGEST = "sha256:c3903fa96db31479c2a28e5d00e05260b039ff856f8b2cca7280c78299877731" # The command that produced the table, per cap (`{cap}` is the cell's memory cap). It runs against an empty data directory, so # the page cache it recommends is the one for a server whose memory is the only input, as for the cells that build their store. NEO4J_RECOMMENDATION_COMMAND = ("docker run --rm -m 2g --entrypoint neo4j-admin neo4j@" + NEO4J_IMAGE_DIGEST