apache · zy-kkk · Sep 23, 2026 · Sep 25, 2026 · Sep 25, 2026 · Sep 25, 2026
@@ -118,10 +118,24 @@ services:
       - ./scripts/lance_rest_server.py:/opt/lance-rest/server.py:ro
     environment:
       LANCE_REST_BEARER_TOKEN: doris-lance-rest-test-token
-      LANCE_REST_TABLES_JSON: '{"all_types":"s3://warehouse/lance/all_types.lance","all_types_unprefixed":"s3://warehouse/lance/all_types.lance"}'
+      LANCE_REST_TABLES_JSON: '{"all_types":"s3://warehouse/lance/all_types.lance","all_types_unprefixed":"s3://warehouse/lance/all_types.lance","time_travel":"s3://warehouse/lance/time_travel.lance"}'
       # all_types_unprefixed serves the same dataset but vends its credentials under the
       # unprefixed object-store spelling, which is what real namespace servers emit.
-      LANCE_REST_UNPREFIXED_TABLES_JSON: '["all_types_unprefixed"]'
+      LANCE_REST_UNPREFIXED_TABLES_JSON: '["all_types_unprefixed","time_travel_managed_unprefixed"]'
+      # All managed tables are the same three-version dataset; they differ in what the namespace
+      # records. A reader that resolves versions through the namespace cannot see a version
+      # missing there even though its manifest is still in storage, and its latest version is
+      # the namespace's latest, not storage's:
+      #   time_travel_managed             every version, with commit times, and branch dev
+      #                                   (versions 2 and 3 under tree/dev/)
+      #   time_travel_managed_partial     versions 1 and 3
+      #   time_travel_managed_lagging     versions 1 and 2, storage already has 3
+      #   time_travel_managed_untimed     versions 1 and 3, no commit times reported
+      #   time_travel_managed_unprefixed  every version, credentials vended unprefixed
+      # Versions and commit times (epoch millis, UTC) match the committed time_travel.lance,
+      # see lance_build_time_travel.py. Doris resolves FOR TIME AS OF from the manifests' commit
+      # times, so the reported ones are not what the time-travel results depend on.
+      LANCE_REST_MANAGED_TABLES_JSON: '{"time_travel_managed":{"uri":"s3://warehouse/lance/time_travel.lance","versions":[{"version":1,"timestamp_millis":1789823167597},{"version":2,"timestamp_millis":1789823169113},{"version":3,"timestamp_millis":1789823170621}],"branches":{"dev":{"versions":[2,3]}}},"time_travel_managed_partial":{"uri":"s3://warehouse/lance/time_travel.lance","versions":[{"version":1,"timestamp_millis":1789823167597},{"version":3,"timestamp_millis":1789823170621}]},"time_travel_managed_lagging":{"uri":"s3://warehouse/lance/time_travel.lance","versions":[{"version":1,"timestamp_millis":1789823167597},{"version":2,"timestamp_millis":1789823169113}]},"time_travel_managed_untimed":{"uri":"s3://warehouse/lance/time_travel.lance","versions":[1,3]},"time_travel_managed_unprefixed":{"uri":"s3://warehouse/lance/time_travel.lance","versions":[{"version":1,"timestamp_millis":1789823167597},{"version":2,"timestamp_millis":1789823169113},{"version":3,"timestamp_millis":1789823170621}]}}'
       LANCE_S3_ACCESS_KEY: admin
       LANCE_S3_SECRET_KEY: password
       LANCE_S3_REGION: us-east-1

@@ -29,6 +29,9 @@
   - __manifest            Directory Namespace V2 manifest table (with its scalar indexes).
   - all_types.lance       The pre-existing compatibility-mode root table, re-registered as-is.
   - nested_null.lance     Nullable Null leaves inside lists, structs, and maps.
+  - time_travel.lance     Three uncompacted versions for FOR VERSION / TIME AS OF; carried
+                          over as-is because the suites hard-code its commit times
+                          (see lance_build_time_travel.py).
   - The `doris` namespace with two full-text-search fixtures, one indexed vector table per cell of the
     algorithm x element type x metric matrix (hash-prefixed directories), listed in
     VECTOR_TABLES below; BREADTH_TABLE, one table carrying the remaining cells at plan
@@ -88,6 +91,7 @@
 import pyarrow.ipc as ipc
 from lance_build_multivector import build as build_multivector, check as check_multivector
 from lance_build_nested_null import build as build_nested_null, check as check_nested_null
+from lance_build_time_travel import check as check_time_travel
 from lance_namespace_urllib3_client.models import (
     CreateNamespaceRequest,
     CreateTableRequest,
@@ -103,6 +107,7 @@
 NAMESPACE = "doris"
 ALL_TYPES_DIR = "all_types.lance"
 NESTED_NULL_DIR = "nested_null.lance"
+TIME_TRAVEL_DIR = "time_travel.lance"
 MANIFEST_DIR = "__manifest"
 
 # 4-bit PQ keeps codebook training comfortable on 1024 rows. This only serves fixture
@@ -931,8 +936,10 @@ def build_multi_frag(root: Path) -> None:
     lance.dataset(location).delete(f"row_id in ({deleted})")
 
 
-def build(root: Path, all_types_source: Path) -> None:
+def build(root: Path, all_types_source: Path, time_travel_source: Path) -> None:
     shutil.copytree(all_types_source, root / ALL_TYPES_DIR)
+    # Not rebuilt: its commit times are hard-coded in the time-travel suites.
+    shutil.copytree(time_travel_source, root / TIME_TRAVEL_DIR)
     build_multi_frag(root)
     # Recreate this fixture in staging because promotion replaces the entire catalog tree.
     build_nested_null(root / NESTED_NULL_DIR)
@@ -1726,6 +1733,7 @@ def check_catalog(root: Path) -> None:
     check_nested_dataset(nested.location)
     check_multi_frag(root)
     check_nested_null(root / NESTED_NULL_DIR)
+    check_time_travel(root / TIME_TRAVEL_DIR)
     check_multivector(root / "multivector.lance")
 
     full_fts = namespace.describe_table(DescribeTableRequest(id=[NAMESPACE, FTS_TABLE]))
@@ -1793,11 +1801,15 @@ def main() -> int:
     if not all_types_source.is_dir():
         print(f"missing all_types source: {all_types_source}", file=sys.stderr)
         return 1
+    time_travel_source = output / TIME_TRAVEL_DIR
+    if not time_travel_source.is_dir():
+        print(f"missing time_travel source: {time_travel_source}", file=sys.stderr)
+        return 1
 
     with tempfile.TemporaryDirectory(prefix="lance_fixture_") as staging_name:
         staging = Path(staging_name) / "lance"
         staging.mkdir()
-        build(staging, all_types_source)
+        build(staging, all_types_source, time_travel_source)
         check_catalog(staging)
         backup = output.with_name(output.name + ".old")
         if backup.exists():

@@ -0,0 +1,177 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Generate the Lance time-travel regression fixture.
+
+time_travel.lance is a root table of the preinstalled Directory catalog next to
+all_types.lance. Every other fixture is compacted to a single version, so this is the one
+dataset whose history survives: three commits, none of them cleaned up.
+
+    version 1  create  row_id 1..3   tag column "v1"   Lance tag v1
+    version 2  append  row_id 4..6   tag column "v2"   Lance tag v2
+    version 3  append  row_id 7..9   tag column "v3"   Lance tag v3
+
+Each version also carries a Lance tag of the same name (stored under _refs/tags/), which the
+suites select with tbl@tag(v2). A branch "dev" (stored under tree/dev/, metadata in
+_refs/branches/) is forked from version 2 and carries one extra commit:
+
+    dev version 3  append  row_id 100  tag column "dev"
+
+so the branch's version 3 has rows 1..6 and 100, while main's version 3 has rows 1..9; the
+suites select it with tbl@branch(dev). The main chain is not touched by the branch. A tag "rel"
+points at the branch's version 3, so tbl@tag(rel) must read the branch, not main's version 3.
+
+A Lance branch is a shallow clone: its manifests record the parent's location as an absolute
+URI (Manifest.base_paths), so a branch only reads where it was created. The committed
+branch was therefore created against the fixture's final location, s3://warehouse/lance/
+time_travel.lance, and synced back into this directory; build() writes the main chain and
+the tags locally, and --create-branch <uri> creates the branch at the uploaded location:
+
+  python3 lance_build_time_travel.py --create-branch s3://warehouse/lance/time_travel.lance \
+      --storage-option endpoint=http://127.0.0.1:19000 --storage-option access_key_id=admin \
+      --storage-option secret_access_key=password --storage-option region=us-east-1 \
+      --storage-option allow_http=true
+  # then sync tree/ and _refs/branches/ from that location into preinstalled_data/lance/time_travel.lance/
+
+check() verifies the branch files and that they point at that URI without opening the branch,
+which is not possible offline.
+
+The commits are spaced apart so that FOR TIME AS OF can land between two of them. Lance
+stores the commit time of each version in its manifest, and that time is whatever wall
+clock this script ran at. Doris has no SQL to read those times back, so the regression
+suites hard-code them (test_lance_time_travel and test_lance_rest_time_travel, together
+with their .out files). Regenerating this dataset therefore means updating those suites
+from the times this script prints; that is why lance_build_preinstalled_catalog.py carries
+the committed directory over as-is, like all_types.lance, instead of rebuilding it, and only
+runs check() on it.
+
+The same directory serves three catalogs in the regression suites: the filesystem catalog,
+the REST catalog with storage-native versions, and the REST catalog with namespace-managed
+versions, where lance_rest_server.py answers the version endpoints from a static list
+matching the versions written here.
+
+Run it with the writer pinned in lance_fixture_requirements.txt; the main script's
+check_pinned_writer() enforces the pin for the whole catalog, this script alone does not.
+
+Usage:
+  python3 lance_build_time_travel.py preinstalled_data/lance/time_travel.lance
+  python3 lance_build_time_travel.py --check preinstalled_data/lance/time_travel.lance
+"""
+import argparse
+import shutil
+import time
+from pathlib import Path
+
+import lance
+import pyarrow as pa
+
+COMMITS = (("create", 1, 3, "v1"), ("append", 4, 6, "v2"), ("append", 7, 9, "v3"))
+BRANCH = "dev"
+BRANCH_ROW_ID = 100
+BRANCH_TAG = "rel"
+BRANCH_PARENT_URI = "s3://warehouse/lance/time_travel.lance"
+COMMIT_GAP_SECONDS = 1.5
+
+
+def rows_of(low: int, high: int, tag: str) -> pa.Table:
+    return pa.table({
+        "row_id": pa.array(range(low, high + 1), pa.int32()),
+        "tag": pa.array([tag] * (high - low + 1), pa.string()),
+    })
+
+
+def build(output: Path) -> None:
+    if output.exists():
+        shutil.rmtree(output)
+    for index, (mode, low, high, tag) in enumerate(COMMITS):
+        if index > 0:
+            time.sleep(COMMIT_GAP_SECONDS)
+        # Match all_types.lance (data storage version 2.2) so every committed Lance data file
+        # shares one on-disk format with the rest of the fixture.
+        lance.write_dataset(rows_of(low, high, tag), str(output), mode=mode,
+                            data_storage_version="2.2")
+    dataset = lance.dataset(str(output))
+    for version, (_, _, _, tag) in zip((1, 2, 3), COMMITS):
+        dataset.tags.create(tag, version)
+    print(f"main chain and tags written; create the branch with --create-branch {BRANCH_PARENT_URI}")
+
+
+def create_branch(uri: str, storage_options: dict) -> None:
+    """Forks the branch at the dataset's final location and appends its extra row there."""
+    dataset = lance.dataset(uri, storage_options=storage_options)
+    assert [v["version"] for v in dataset.versions()] == [1, 2, 3], "upload the main chain first"
+    dataset.create_branch(BRANCH, 2)
+    lance.write_dataset(rows_of(BRANCH_ROW_ID, BRANCH_ROW_ID, BRANCH), f"{uri}/tree/{BRANCH}",
+                        mode="append", data_storage_version="2.2", storage_options=storage_options)
+    branch = lance.dataset(f"{uri}/tree/{BRANCH}", storage_options=storage_options)
+    assert branch.version == 3 and sorted(branch.to_table()["row_id"].to_pylist()) == [1, 2, 3, 4, 5, 6, BRANCH_ROW_ID]
+    dataset.tags.create(BRANCH_TAG, (BRANCH, 3))
+    print(f"branch {BRANCH} and tag {BRANCH_TAG} created at {uri}; sync tree/, _refs/branches/ and _refs/tags/ back")
+
+
+def check(output: Path) -> None:
+    dataset = lance.dataset(str(output))
+    versions = dataset.versions()
+    assert [v["version"] for v in versions] == [1, 2, 3], (
+        f"time-travel fixture must keep exactly versions 1..3: {versions}")
+    timestamps = [v["timestamp"] for v in versions]
+    assert timestamps == sorted(timestamps) and len(set(timestamps)) == 3, (
+        f"time-travel fixture commit times must be distinct and increasing: {timestamps}")
+    assert all((b - a).total_seconds() >= 1 for a, b in zip(timestamps, timestamps[1:])), (
+        f"time-travel fixture commits must be at least one second apart: {timestamps}")
+    for version, (_, _, high, tag) in zip((1, 2, 3), COMMITS):
+        table = dataset.checkout_version(version).to_table().sort_by("row_id")
+        assert table["row_id"].to_pylist() == list(range(1, high + 1)), (
+            f"version {version} rows differ from expected: {table}")
+        assert table["tag"].to_pylist()[-1] == tag, f"version {version} tag differs: {table}"
+    tags = {name: (ref["branch"], ref["version"]) for name, ref in dataset.tags.list().items()}
+    assert tags == {"v1": (None, 1), "v2": (None, 2), "v3": (None, 3), BRANCH_TAG: (BRANCH, 3)}, (
+        f"time-travel fixture tags differ: {tags}")
+    assert list(dataset.branches.list()) == [BRANCH], f"time-travel fixture branches differ: {dataset.branches.list()}"
+    branch_manifests = sorted((output / "tree" / BRANCH / "_versions").glob("*.manifest"))
+    assert len(branch_manifests) == 2, f"branch {BRANCH} must carry versions 2 and 3: {branch_manifests}"
+    for manifest in branch_manifests:
+        assert BRANCH_PARENT_URI.encode() in manifest.read_bytes(), (
+            f"{manifest} must reference the parent at {BRANCH_PARENT_URI}; a branch created elsewhere is unreadable there")
+    for version in versions:
+        print(f"time_travel.lance version {version['version']} committed at "
+              f"{version['timestamp'].isoformat()}")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("output", type=Path, nargs="?",
+                        help="path of time_travel.lance (not used with --create-branch)")
+    parser.add_argument("--check", action="store_true", help="verify the existing fixture")
+    parser.add_argument("--create-branch", metavar="URI",
+                        help="create the dev branch at the uploaded dataset URI instead of building")
+    parser.add_argument("--storage-option", action="append", default=[], metavar="KEY=VALUE",
+                        help="Lance storage option for --create-branch (repeatable)")
+    args = parser.parse_args()
+    if not args.create_branch and args.output is None:
+        parser.error("the output path is required unless --create-branch is given")
+    if args.create_branch:
+        create_branch(args.create_branch, dict(item.split("=", 1) for item in args.storage_option))
+    elif args.check:
+        check(args.output)
+    else:
+        build(args.output)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -19,11 +19,11 @@
 #
 # The fixture has several readers and they do not all run the same Lance version:
 #
-#   lance-c v0.1.6 (lance-rs 7.0.0-beta.7)  BE, both from source and from the prebuilt
+#   lance-c v0.1.9 (lance-rs 11.0.0, patched)  BE, both from source and from the prebuilt
 #                                           doris-thirdparty package - thirdparty/vars.sh
 #   lance-java 4.0.0                        Spark, via lance-spark-bundle 0.4.0; it
 #                                           registers runtime tables into the same __manifest
-#   lance-java 9.1.0-beta.3                 Doris FE Directory Namespace client - fe/pom.xml
+#   lance-java 12.0.0                       Doris FE Directory Namespace client - fe/pom.xml
 #
 # The writer is pinned to the Lance generation the BE actually reads. It used to be pinned
 # to the oldest reader instead, on the theory that an older writer is readable by every