From 597015306316a3e34905185e97468bce0f20a9b5 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Fri, 2 Oct 2026 20:58:43 +0900 Subject: [PATCH] bindings: one-way edge refusal and query contract (#8625), warm_up() for the vector index (#8852) (#87) ArcadeData/arcadedb#8625 (fixed in #8628, in 26.10.1): graph_batch with bidirectional=False on an edge type declared two-way is now refused, new_edge raising ArcadeDBError naming the type, and nothing is written. The docs and the graph_batch() docstring said the batch does not check; they now say refusal. docs/guide/graphs.md gains the one-way query contract, each claim checked on the synced wheel: patterns (Cypher from either end and undirected, SQL MATCH) see every edge; in(), inE(), both(), bothE(), and the target's get_in_edges() see none; out() and get_out_edges() on the source see them all. ArcadeData/arcadedb#8852 (fixed in #8863): VectorIndex.warm_up() calls LSMVectorIndex.warmUp() on every underlying index, loading the persisted graph now instead of on the first search; ArcadeDBError on an engine without it. Documented in the API and the guide beside the throwaway search, whose example wraps the vector in a list (a lone list argument is the parameter list). Tests, on the synced engine 802233cb57 (wheel built from its image's jars) and against arcadedb-embedded==26.9.1, where all three fail for the right reason: test_graph_batch_refuses_one_way_edges_in_a_two_way_type (26.9.1 accepts the edge), test_one_way_edges_are_seen_by_patterns_not_by_in (26.9.1's incoming Cypher pattern counts 0 of 100), and test_warm_up_loads_the_persisted_graph_before_the_first_search (graphState 0 and graphNodeCount 0 after the reopen, all 2,000 nodes resident after warm_up; 26.9.1 has no warm_up). Full suite 492 passed, 3 skipped; bandit at low/low and pre-commit clean. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Lj5isEaoaGCeaZe5nNoWX4 --- bindings/python/docs/api/graph_batch.md | 8 +- bindings/python/docs/api/vector.md | 38 ++++++ .../development/testing/test-graph-batch.md | 19 +++ .../docs/development/testing/test-vector.md | 1 + bindings/python/docs/guide/graphs.md | 23 +++- bindings/python/docs/guide/vectors.md | 4 + bindings/python/src/arcadedb_embedded/core.py | 7 +- .../python/src/arcadedb_embedded/vector.py | 34 ++++++ bindings/python/tests/test_graph_batch.py | 112 ++++++++++++++++++ bindings/python/tests/test_vector.py | 39 ++++++ 10 files changed, 276 insertions(+), 9 deletions(-) diff --git a/bindings/python/docs/api/graph_batch.md b/bindings/python/docs/api/graph_batch.md index 3b9e1bd761a..001ccb63982 100644 --- a/bindings/python/docs/api/graph_batch.md +++ b/bindings/python/docs/api/graph_batch.md @@ -38,9 +38,11 @@ Create a configured batch helper tied to the current database. - `light_edges`: create property-less light edges when appropriate - `bidirectional`: store each edge on both vertices (the default) or on its source only. Pass `False` only for an edge type declared one-way (`CREATE EDGE TYPE ... - UNIDIRECTIONAL`). The batch does not check: one-way edges loaded into a two-way type - (the default `CREATE EDGE TYPE`) make any query the planner walks from the target end - return 0 rows with no error (ArcadeData/arcadedb#8625) + UNIDIRECTIONAL`). From 26.10.1 a one-way edge in a two-way type (the default + `CREATE EDGE TYPE`) is refused: `new_edge` raises `ArcadeDBError` naming the type and + writes nothing. Before 26.10.1 it was accepted, and every query the planner walked from + the target end returned 0 rows with no error (ArcadeData/arcadedb#8625). What a one-way + edge is visible to is in [Graphs](../guide/graphs.md) - `commit_every`: commit cadence during batch work - `use_wal`: write-ahead log during the import. **Off by default**: a crash in the middle of the import can lose its tail, with nothing to replay. Pass diff --git a/bindings/python/docs/api/vector.md b/bindings/python/docs/api/vector.md index dc6f47e6158..fb80592444c 100644 --- a/bindings/python/docs/api/vector.md +++ b/bindings/python/docs/api/vector.md @@ -612,6 +612,44 @@ cache far better than fp32 does. --- +### `VectorIndex.warm_up()` + +Load the index's graph now instead of on the first search. From 26.10.1. + +After a database is opened, a vector index loads its persisted graph lazily, on +the first search, and that search pays for it (ArcadeData/arcadedb#8852). After +the 26.10.1 fix that is about 1 s at 1M 64-dimension vectors, growing with the +index, and more on an index with deletions since its graph was saved, which +still re-reads every vector's document. A service that restarts can call +`warm_up()` right after opening the database, so the first user query does not +pay it. + +It loads the existing graph and does not rebuild it (that is +`build_graph_now()`). It is a no-op once the graph is in memory, and safe to call +while other threads search. On an engine before 26.10.1 it raises +`ArcadeDBError`; there, one throwaway search does the same. + +`get_stats()["graphState"]` reads `0` (loading) between the open and the first +search or `warm_up()`, and `graphNodeCount` is `0` until the graph is resident. + +**Returns:** + +- `None` + +**Example:** + +```python +with arcadedb.open_database("./vectors") as db: + index = db.schema.get_vector_index("Doc", "embedding") + index.warm_up() # pay the graph load now, not on the first query + +# On engines before 26.10.1, the same with one throwaway search. The vector is +# wrapped in a list: a lone list argument is read as the parameter list itself. +# db.query("sql", "SELECT vectorNeighbors('Doc[embedding]', ?, 1)", [probe_vector]) +``` + +--- + ### `VectorIndex.build_graph_now()` Force an immediate rebuild/preparation of the vector graph. diff --git a/bindings/python/docs/development/testing/test-graph-batch.md b/bindings/python/docs/development/testing/test-graph-batch.md index 952de0cddd9..e7b8696cc44 100644 --- a/bindings/python/docs/development/testing/test-graph-batch.md +++ b/bindings/python/docs/development/testing/test-graph-batch.md @@ -42,6 +42,25 @@ wording rather than grepping for the parameter name: an unwired keyword raises contains the parameter name, so a name-substring check passes on exactly the broken code it is meant to catch. +### 7) a one-way edge in a two-way type is refused + +`test_graph_batch_refuses_one_way_edges_in_a_two_way_type`: with +`bidirectional=False`, `new_edge` on an edge type declared two-way (the +`CREATE EDGE TYPE` default) raises `ArcadeDBError` naming the type and the word +"bidirectional", and the type holds 0 edges; on a type declared +`UNIDIRECTIONAL` the same batch writes its edge (1). Fails on 26.9.1, which +accepted the edge (ArcadeData/arcadedb#8625). + +### 8) what sees a one-way edge + +`test_one_way_edges_are_seen_by_patterns_not_by_in`: 50 questions each tagged +twice through a `UNIDIRECTIONAL` type (100 edges). Cypher counts 100 written +from the source, from the target, and undirected; SQL `MATCH` from the target +returns 100 rows; `in()`, `inE()`, `both()`, and `bothE()` from the target +return 0, `out()` from the source 100, and the target vertex's +`get_in_edges()` is empty. Fails on 26.9.1, where the incoming Cypher pattern +returned 0. + ## Why It Matters `GraphBatch` is the repository's preferred bulk graph-ingest path from Python, so these tests protect the performance-oriented API surface and its configuration validation behavior. diff --git a/bindings/python/docs/development/testing/test-vector.md b/bindings/python/docs/development/testing/test-vector.md index 515bdd776c1..426d86b2965 100644 --- a/bindings/python/docs/development/testing/test-vector.md +++ b/bindings/python/docs/development/testing/test-vector.md @@ -55,6 +55,7 @@ What the tests cover: - **test_lsm_vector_search_ef_search**: `find_nearest(k=2, ef_search=32)` returns 2 results with the closest vector first. - **test_lsm_vector_search_rejects_invalid_ef_search**: `ef_search=0` raises `ArcadeDBError` mentioning `ef_search`. - **test_lsm_vector_build_graph_now**: after `build_graph_now()`, a search returns the closest vector. +- **test_warm_up_loads_the_persisted_graph_before_the_first_search**: 2,000 vectors, index built, database reopened: `get_stats()` reads `graphState` 0 and `graphNodeCount` 0; after `warm_up()` the state is not 0 and all 2,000 nodes are resident; a second `warm_up()` changes nothing, and the search for a stored vector returns it at distance 0.0. Fails on 26.9.1, which has no `warm_up()` (ArcadeData/arcadedb#8852). - **test_lsm_vector_delete_and_search_others**: of 100 random vectors, every tenth is deleted; each deleted vector is absent from its own search and each remaining one is found. - **test_lsm_vector_search_comprehensive**: on small word embeddings, the neighbours of `king` include `queen` and `man` or `woman` but not `cat` or `dog`, and the neighbours of `cat` include `dog` but not `king`. - **test_document_vector_search**: search on a document type returns `apple` and `banana` (not `car`) for one query and `car` and `truck` (not `apple`) for another; results are `MyDoc` records with 4-dimensional embeddings. diff --git a/bindings/python/docs/guide/graphs.md b/bindings/python/docs/guide/graphs.md index a316fff1f6b..8109dc0ad1e 100644 --- a/bindings/python/docs/guide/graphs.md +++ b/bindings/python/docs/guide/graphs.md @@ -36,9 +36,26 @@ as a comparison arm, pinned to one worker and checked against what it submitted. **Edge direction must match the schema.** `CREATE EDGE TYPE` makes a two-way type by default, and `GraphBatch` stores both directions unless you pass `bidirectional=False`. -Pass it only for a type declared one-way (`CREATE EDGE TYPE ... UNIDIRECTIONAL`): the -batch does not check, and one-way edges in a two-way type make any query the planner -walks from the target end return 0 rows with no error (`ArcadeData/arcadedb#8625`). +Pass it only for a type declared one-way (`CREATE EDGE TYPE ... UNIDIRECTIONAL`). From +26.10.1 the engine refuses a one-way edge in a two-way type: `new_edge` raises +`ArcadeDBError` naming the type, and nothing is written. Before 26.10.1 the batch did not +check, and such edges made any query the planner walked from the target end return 0 +rows with no error (`ArcadeData/arcadedb#8625`, fixed in #8628). + +**What sees a one-way edge.** An edge of a `UNIDIRECTIONAL` type is stored on its source +vertex only. From 26.10.1: + +- Patterns see every such edge, whichever way they are written: Cypher + (`(t:Tag)<-[:TAGGED_WITH]-(q:Question)`, `(q)-[:TAGGED_WITH]->(t)`, and the undirected + `(t)-[:TAGGED_WITH]-(q)`) and SQL `MATCH`. The planner walks from the source when it can; + otherwise it scans the edge type once per query and reuses that scan. +- `in()`, `inE()`, `both()`, and `bothE()` in SQL, and the vertex API + (`get_in_edges()`, `get_both_edges()` on the target), read what the target vertex stores, + which for a one-way edge is nothing. `out()` and `get_out_edges()` on the source see it. + +Before 26.10.1, patterns walked from the target end returned 0 rows as well. If a type +is often queried from its target, declare it two-way (the default): a pattern from the +target of a one-way type pays a scan of the whole edge type. ## Overview diff --git a/bindings/python/docs/guide/vectors.md b/bindings/python/docs/guide/vectors.md index 474b4c5e984..ce0e3075e5f 100644 --- a/bindings/python/docs/guide/vectors.md +++ b/bindings/python/docs/guide/vectors.md @@ -78,6 +78,10 @@ Preferred split: - `get_stats()` returns live counters (cache hits and misses, where vectors were read from, graph state) for sizing caches and diagnosing build cost. See [Vector Caches](../api/vector.md#vector-caches). +- After a database is opened, an index loads its graph on the first search, and that + search pays for it. A service that restarts can call `warm_up()` on the loaded index + right after opening (26.10.1 and later), or run one throwaway search on older engines. + See [`VectorIndex.warm_up()`](../api/vector.md#vectorindexwarm_up). ## Distance Functions (scoring behavior) diff --git a/bindings/python/src/arcadedb_embedded/core.py b/bindings/python/src/arcadedb_embedded/core.py index fb83a326e0e..df2c25d4cc3 100644 --- a/bindings/python/src/arcadedb_embedded/core.py +++ b/bindings/python/src/arcadedb_embedded/core.py @@ -913,9 +913,10 @@ def graph_batch( edge_list_initial_size: Initial edge-segment size in bytes. light_edges: Create property-less edges as light edges when True. bidirectional: Connect incoming edges as well as outgoing edges (the - default). Pass False only for an edge type declared UNIDIRECTIONAL: the - batch does not check, and one-way edges in a two-way type make any query - the planner walks from the target end return 0 rows + default). Pass False only for an edge type declared UNIDIRECTIONAL. From + 26.10.1 a one-way edge in a two-way type is refused: ``new_edge`` raises + ArcadeDBError naming the type. Before 26.10.1 it was accepted, and any + query the planner walked from the target end returned 0 rows (ArcadeData/arcadedb#8625). commit_every: Commit cadence within a flush. `0` means one commit per flush. use_wal: Write-ahead log during the import. Off by default, so a crash mid-import can lose its diff --git a/bindings/python/src/arcadedb_embedded/vector.py b/bindings/python/src/arcadedb_embedded/vector.py index b55292434c8..b3e5f6b8514 100644 --- a/bindings/python/src/arcadedb_embedded/vector.py +++ b/bindings/python/src/arcadedb_embedded/vector.py @@ -695,6 +695,40 @@ def get_quantization(self): except Exception: return "NONE" + def warm_up(self): + """ + Load the vector graph now instead of on the first search. + + After a database is opened, the index loads its persisted graph lazily, + on the first search, and that search pays the load (ArcadeData/arcadedb#8852: + about 1 s at 1M 64-dimension vectors on 26.10.1, growing with the index, and + more on an index with deletions since its graph was saved, which still + re-reads every vector's document). Calling this right after + opening the database pays it before the first query arrives, which is + what a service that restarts wants. It loads the existing graph and does + not rebuild it; a no-op when the graph is already in memory, and safe to + call while other threads search. For TypeIndex wrappers, warms every + underlying LSMVectorIndex. + + The alternative on engines before 26.10.1 is one throwaway search. + + Raises: + ArcadeDBError: the index cannot be warmed, including on an engine + older than 26.10.1, which has no ``warmUp()``. + """ + try: + warmed_any = False + for index in self._iter_lsm_indexes(): + index.warmUp() + warmed_any = True + if warmed_any: + return + raise ArcadeDBError("Underlying index is not an LSM vector index") + except ArcadeDBError: + raise + except Exception as e: + raise ArcadeDBError(f"Failed to warm up vector index: {e}") from e + def build_graph_now(self): """ Trigger an immediate rebuild of the underlying vector graph. diff --git a/bindings/python/tests/test_graph_batch.py b/bindings/python/tests/test_graph_batch.py index d57ac155de4..4178d02017c 100644 --- a/bindings/python/tests/test_graph_batch.py +++ b/bindings/python/tests/test_graph_batch.py @@ -200,3 +200,115 @@ def test_graph_batch_invalid_knob_values_are_rejected(temp_db_path): assert "must be" in str(exc), (kwargs, str(exc)) else: raise AssertionError(f"Expected {kwargs} to be rejected") + + +def test_graph_batch_refuses_one_way_edges_in_a_two_way_type(temp_db_path): + """A batch built with `bidirectional=False` stores each edge on its source + only. Before 26.10.1 it did so even for an edge type declared two-way (the + `CREATE EDGE TYPE` default), and every query the planner then walked from the + target end returned 0 rows with no error (ArcadeData/arcadedb#8625, fixed in + #8628). Now the engine refuses the edge, naming the type, and writes nothing; + a type declared UNIDIRECTIONAL loads as before.""" + with arcadedb.create_database(temp_db_path) as db: + db.command("sql", "CREATE VERTEX TYPE Question") + db.command("sql", "CREATE VERTEX TYPE Tag") + db.command("sql", "CREATE EDGE TYPE TaggedWith") + db.command("sql", "CREATE EDGE TYPE OneWay UNIDIRECTIONAL") + + try: + with db.graph_batch(bidirectional=False) as batch: + q = batch.create_vertex("Question", k=1) + t = batch.create_vertex("Tag", k=2) + batch.new_edge(q, "TaggedWith", t) + except arcadedb.ArcadeDBError as exc: + assert "TaggedWith" in str(exc), str(exc) + assert "bidirectional" in str(exc), str(exc) + else: + raise AssertionError( + "a one-way edge in a two-way type was accepted (ArcadeData/arcadedb#8625)" + ) + assert ( + db.query("sql", "SELECT count(*) AS n FROM TaggedWith").first().get("n") + == 0 + ) + + with db.graph_batch(bidirectional=False) as batch: + q = batch.create_vertex("Question", k=3) + t = batch.create_vertex("Tag", k=4) + batch.new_edge(q, "OneWay", t) + assert db.query("sql", "SELECT count(*) AS n FROM OneWay").first().get("n") == 1 + + +def test_one_way_edges_are_seen_by_patterns_not_by_in(temp_db_path): + """The query contract of an edge type declared UNIDIRECTIONAL (26.10.1, + ArcadeData/arcadedb#8625 fixed in #8628). Its edges are stored on the source + vertex only. Patterns, Cypher and SQL MATCH, return every edge whichever + way they are written, walking from the source or scanning the edge type + once; `in()`, `inE()`, `both()`, and the vertex API read what the target + vertex stores, which is nothing. Before 26.10.1 a pattern walked from the + target returned 0 rows too.""" + with arcadedb.create_database(temp_db_path) as db: + db.command("sql", "CREATE VERTEX TYPE Question") + db.command("sql", "CREATE VERTEX TYPE Tag") + db.command("sql", "CREATE EDGE TYPE TaggedWith UNIDIRECTIONAL") + with db.graph_batch(bidirectional=False) as batch: + tags = [batch.create_vertex("Tag", k=i) for i in range(5)] + for i in range(50): + q = batch.create_vertex("Question", k=i) + batch.new_edge(q, "TaggedWith", tags[i % 5]) + batch.new_edge(q, "TaggedWith", tags[(i + 1) % 5]) + + def count(language, query): + return db.query(language, query).first().get("n") + + # Patterns see all 100 edges, from either end. + assert ( + count( + "opencypher", + "MATCH (q:Question)-[:TaggedWith]->(t:Tag) RETURN count(*) AS n", + ) + == 100 + ) + assert ( + count( + "opencypher", + "MATCH (t:Tag)<-[:TaggedWith]-(q:Question) RETURN count(*) AS n", + ) + == 100 + ) + assert ( + count( + "opencypher", + "MATCH (t:Tag)-[:TaggedWith]-(q:Question) RETURN count(*) AS n", + ) + == 100 + ) + assert ( + len( + db.query( + "sql", + "MATCH {type: Tag, as: t}<-TaggedWith-{type: Question, as: q} RETURN t, q", + ).to_list() + ) + == 100 + ) + + # The traversal functions and the vertex API read the target's own + # pointers, which a one-way edge does not write. + for fn in ("in", "inE", "both", "bothE"): + assert ( + count( + "sql", + f"SELECT count(*) AS n FROM (SELECT expand({fn}('TaggedWith')) FROM Tag)", # nosec B608 - fixed function names, no input + ) + == 0 + ), fn + assert ( + count( + "sql", + "SELECT count(*) AS n FROM (SELECT expand(out('TaggedWith')) FROM Question)", + ) + == 100 + ) + tag = db.query("sql", "SELECT FROM Tag WHERE k = 0").first().get_vertex() + assert tag.get_in_edges("TaggedWith") == [] diff --git a/bindings/python/tests/test_vector.py b/bindings/python/tests/test_vector.py index 9fb010c7346..6d4a8d9ac3a 100644 --- a/bindings/python/tests/test_vector.py +++ b/bindings/python/tests/test_vector.py @@ -1811,3 +1811,42 @@ def test_graph_storage_with_quantization(self, test_db): pytest.fail( f"Failed to create index with graph storage and quantization: {e}" ) + + +def test_warm_up_loads_the_persisted_graph_before_the_first_search(tmp_path): + """After a reopen the index loads its persisted graph on the first search, + which pays for it (ArcadeData/arcadedb#8852). `warm_up()` loads it now, so + the first query does not: asserted on the engine's own counters rather than + on a time, which is noisy at test size. On engines before 26.10.1 there is + no warmUp() and the call raises.""" + import random + + db_path = str(tmp_path / "warm_up") + rnd = random.Random(3) # nosec B311 - deterministic test corpus, not security + vecs = [[rnd.random() for _ in range(16)] for _ in range(2000)] + with arcadedb.create_database(db_path) as db: + db.command("sql", "CREATE VERTEX TYPE Doc") + db.command("sql", "CREATE PROPERTY Doc.embedding ARRAY_OF_FLOATS") + with db.transaction(): + for i, v in enumerate(vecs): + # Two parameters: a lone list argument is read as the + # parameter list itself, binding its first float. + db.command("sql", "INSERT INTO Doc SET k = ?, embedding = ?", i, v) + db.create_vector_index("Doc", "embedding", dimensions=16, quantization="NONE") + + with arcadedb.open_database(db_path) as db: + index = db.schema.get_vector_index("Doc", "embedding") + before = index.get_stats() + # Opened, not yet loaded: graphState LOADING (0) and nothing resident. + assert before["graphState"] == 0, before["graphState"] + assert before["graphNodeCount"] == 0, before["graphNodeCount"] + + index.warm_up() + after = index.get_stats() + assert after["graphState"] != 0, after["graphState"] + assert after["graphNodeCount"] == len(vecs), after["graphNodeCount"] + + index.warm_up() # a no-op once loaded + nearest = index.find_nearest(vecs[7], k=1) + assert nearest[0][1] == 0.0, nearest + assert index.get_stats()["graphNodeCount"] == len(vecs)