atelet, nodepath: separate per-actor snapshot directories under /var/lib/ate/snapshots

README.md

@@ -258,6 +258,7 @@ We provide several sample applications demonstrating Agent Substrate's capabilit
 * [Request Parking](docs/request-parking.md): How the router parks requests through transient worker-pool saturation.
 * [Sandbox Asset Cache](docs/sandbox-asset-cache.md): How a node caches the sandbox binaries actors run on, decides which runtimes it needs, and evicts to a budget.
 * [Upgrade Runbook](docs/upgrade.md): Upgrade a running install to a newer supported release, and roll back if needed.
+* [Node Storage](docs/node-storage.md): How to give snapshot I/O its own disk.
 * [Security Guide](docs/security.md): Security overview, security principles, and threat modeling.
 * [Roadmap](docs/roadmap.md): Current limitations and what is planned next.
 * [Benchmarking Guide](benchmarking/README.md): Locust-based load tests, monitoring stack, and the orchestrated benchmark harness.

cmd/atelet/internal/ateletpath/ateletpath.go

@@ -60,8 +60,9 @@ var (
 	// snapshots many actors are created from (see
 	// cmd/atelet/internal/filecache). It lives under BasePath because cache
 	// hits are served as hard links into the per-actor restore dirs under
-	// ActorsDir, and link(2) requires both ends on one mounted filesystem,
-	// the same mount, not merely the same disk.
+	// SnapshotsDir, and link(2) requires both ends on one mounted filesystem,
+	// the same mount, not merely the same disk. With SnapshotsDir on its own
+	// mount, hits are served as copies instead.
 	SnapshotCacheDir = filepath.Join(nodepath.BasePath, "snapshot-cache")
 )
 
@@ -72,6 +73,16 @@ func ActorPath(actorUID string) string {
 	)
 }
 
+// ActorSnapshotPath is the actor's snapshot staging directory, which may be on
+// a different filesystem from ActorPath. Checkpoints rename files between its
+// subdirectories, so they must stay on one filesystem.
+func ActorSnapshotPath(actorUID string) string {
+	return filepath.Join(
+		nodepath.SnapshotsDir,
+		actorUID,
+	)
+}
+
 // ActorSandboxAssetsFile is the per-actor file where atelet records the sandbox
 // binaries (class + content-addressed asset set, for this node's architecture)
 // the actor is currently running. It is written at Run/Restore and read at
@@ -131,14 +142,14 @@ func OCIBundlePath(actorUID, containerName string) string {
 
 func CheckpointStateDir(actorUID string) string {
 	return filepath.Join(
-		ActorPath(actorUID),
+		ActorSnapshotPath(actorUID),
 		"checkpoint-state",
 	)
 }
 
 func LocalCheckpointsDir(actorUID string) string {
 	return filepath.Join(
-		ActorPath(actorUID),
+		ActorSnapshotPath(actorUID),
 		"local-checkpoint",
 	)
 }
@@ -210,7 +221,7 @@ func SystemInfoVolumeRoot(actorUID, volumeName string) string {
 // yet been loaded.
 func RestoreStateDir(actorUID string) string {
 	return filepath.Join(
-		ActorPath(actorUID),
+		ActorSnapshotPath(actorUID),
 		"restore-state",
 	)
 }

cmd/atelet/internal/ateletpath/ateletpath_test.go

@@ -26,24 +26,41 @@ func TestActorDirs(t *testing.T) {
 	if actorDirs.GetRootDir() != ActorPath(actorUID) {
 		t.Fatalf("root_dir = %q, want %q", actorDirs.GetRootDir(), ActorPath(actorUID))
 	}
-	under := map[string]string{
+	snapshotRoot := ActorSnapshotPath(actorUID)
+	// Disjoint trees: neither root may sit inside (or equal) the other.
+	for _, p := range [][2]string{{snapshotRoot, actorDirs.GetRootDir()}, {actorDirs.GetRootDir(), snapshotRoot}} {
+		if rel, err := filepath.Rel(p[1], p[0]); err == nil && !strings.HasPrefix(rel, "..") {
+			t.Fatalf("%q is inside %q; ActorSnapshotPath and ActorPath must be disjoint", p[0], p[1])
+		}
+	}
+
+	underActorRoot := map[string]string{
 		"oci_bundle_dir":                actorDirs.GetOciBundleDir(),
-		"checkpoint_dir":                actorDirs.GetCheckpointDir(),
-		"restore_dir":                   actorDirs.GetRestoreDir(),
 		"durable_dir_volume_mounts_dir": actorDirs.GetDurableDirVolumeMountsDir(),
 		"system_info_volume_roots_dir":  actorDirs.GetSystemInfoVolumeRootsDir(),
 		"external_volume_mounts_dir":    actorDirs.GetExternalVolumeMountsDir(),
 		"sandbox_assets_dir":            actorDirs.GetSandboxAssetsDir(),
 	}
+	underSnapshotRoot := map[string]string{
+		"checkpoint_dir":        actorDirs.GetCheckpointDir(),
+		"restore_dir":           actorDirs.GetRestoreDir(),
+		"local_checkpoints_dir": LocalCheckpointsDir(actorUID),
+		"local_snapshot_dir":    LocalSnapshotDir(actorUID, "snap-1"),
+	}
+
 	seen := map[string]string{}
-	for field, dir := range under {
-		rel, err := filepath.Rel(actorDirs.GetRootDir(), dir)
-		if err != nil || rel == "." || strings.HasPrefix(rel, "..") {
-			t.Errorf("%s = %q is not a proper subdirectory of root_dir %q", field, dir, actorDirs.GetRootDir())
-		}
-		if other, dup := seen[dir]; dup {
-			t.Errorf("%s and %s share %q", field, other, dir)
+	checkSubdirs := func(parentName, parent string, dirs map[string]string) {
+		for field, dir := range dirs {
+			rel, err := filepath.Rel(parent, dir)
+			if err != nil || rel == "." || strings.HasPrefix(rel, "..") {
+				t.Errorf("%s = %q is not a proper subdirectory of %s %q", field, dir, parentName, parent)
+			}
+			if other, dup := seen[dir]; dup {
+				t.Errorf("%s and %s share %q", field, other, dir)
+			}
+			seen[dir] = field
 		}
-		seen[dir] = field
 	}
+	checkSubdirs("root_dir", actorDirs.GetRootDir(), underActorRoot)
+	checkSubdirs("snapshot_root", snapshotRoot, underSnapshotRoot)
 }

cmd/atelet/internal/snapshottracker/background_test.go

@@ -34,12 +34,14 @@ import (
 )
 
 // useActorDir roots atelet's on-node state in temp directories and creates the
-// tracked actor's directory, so the tracker writes its snapshot state file.
+// tracked actor's directories, so the tracker writes its snapshot state file.
 func useActorDir(t *testing.T) {
 	t.Helper()
 	useTempNodeDirs(t)
-	if err := os.MkdirAll(ateletpath.ActorPath(trackedActorUID), 0o700); err != nil {
-		t.Fatal(err)
+	for _, dir := range []string{ateletpath.ActorPath(trackedActorUID), ateletpath.ActorSnapshotPath(trackedActorUID)} {
+		if err := os.MkdirAll(dir, 0o700); err != nil {
+			t.Fatal(err)
+		}
 	}
 }
 

cmd/atelet/internal/snapshottracker/tracker_test.go

@@ -47,10 +47,11 @@ const trackedActorUID = "actor-uid-1"
 func useTempNodeDirs(t *testing.T) {
 	t.Helper()
 	root := t.TempDir()
-	origActors := nodepath.ActorsDir
+	origActors, origSnapshots := nodepath.ActorsDir, nodepath.SnapshotsDir
 	nodepath.ActorsDir = filepath.Join(root, "actors")
+	nodepath.SnapshotsDir = filepath.Join(root, "snapshots")
 	t.Cleanup(func() {
-		nodepath.ActorsDir = origActors
+		nodepath.ActorsDir, nodepath.SnapshotsDir = origActors, origSnapshots
 	})
 }
 

cmd/atelet/lifecycle_linux_test.go

@@ -0,0 +1,51 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"errors"
+	"path/filepath"
+	"syscall"
+	"testing"
+
+	"go.ate.dev/substrate/internal/nodepath"
+	"go.ate.dev/substrate/internal/proto/ateletpb"
+)
+
+// TestLocalSnapshotGCWithSeparateSnapshotsFilesystem runs TestLocalSnapshotGC
+// with SnapshotsDir on a different filesystem from ActorsDir.
+func TestLocalSnapshotGCWithSeparateSnapshotsFilesystem(t *testing.T) {
+	for _, fidelity := range []ateletpb.SnapshotFidelity{
+		ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES,
+		ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS,
+		ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY,
+	} {
+		t.Run(fidelity.String(), func(t *testing.T) {
+			runLocalSnapshotGC(t, fidelity, useSeparateSnapshotsFilesystem)
+		})
+	}
+}
+
+// useSeparateSnapshotsFilesystem is useTempNodeDirs with SnapshotsDir on another mount.
+func useSeparateSnapshotsFilesystem(t *testing.T) {
+	t.Helper()
+	useTempNodeDirs(t)
+	shm := otherMount(t)
+	root := filepath.Dir(nodepath.ActorsDir)
+	if err := canHardLink(root, shm); !errors.Is(err, syscall.EXDEV) {
+		t.Fatalf("canHardLink(%s, %s) = %v, want EXDEV: SnapshotsDir must be on another mount than ActorsDir", root, shm, err)
+	}
+	nodepath.SnapshotsDir = filepath.Join(shm, "snapshots")
+}

cmd/atelet/lifecycle_test.go

@@ -58,10 +58,11 @@ import (
 func useTempNodeDirs(t *testing.T) {
 	t.Helper()
 	root := t.TempDir()
-	origActors := nodepath.ActorsDir
+	origActors, origSnapshots := nodepath.ActorsDir, nodepath.SnapshotsDir
 	nodepath.ActorsDir = filepath.Join(root, "actors")
+	nodepath.SnapshotsDir = filepath.Join(root, "snapshots")
 	t.Cleanup(func() {
-		nodepath.ActorsDir = origActors
+		nodepath.ActorsDir, nodepath.SnapshotsDir = origActors, origSnapshots
 	})
 }
 
@@ -215,12 +216,14 @@ func TestLocalSnapshotGC(t *testing.T) {
 		ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS,
 		ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY,
 	} {
-		t.Run(fidelity.String(), func(t *testing.T) { runLocalSnapshotGC(t, fidelity) })
+		t.Run(fidelity.String(), func(t *testing.T) { runLocalSnapshotGC(t, fidelity, useTempNodeDirs) })
 	}
 }
 
-func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity) {
-	useTempNodeDirs(t)
+// runLocalSnapshotGC runs the TestLocalSnapshotGC flow on the node dirs useDirs
+// sets up.
+func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity, useDirs func(*testing.T)) {
+	useDirs(t)
 	ctx := t.Context()
 
 	const (
@@ -391,16 +394,8 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity) {
 
 	// Terminate is the only chance to reclaim the actor's directory: nothing
 	// else on the node deletes it.
-	actorDir := ateletpath.ActorPath(actorUID)
-	if entries, err := os.ReadDir(actorDir); err == nil {
-		left := make([]string, 0, len(entries))
-		for _, e := range entries {
-			left = append(left, e.Name())
-		}
-		t.Errorf("actor dir %s survived terminate with %d entries: %v", actorDir, len(left), left)
-	} else if !os.IsNotExist(err) {
-		t.Errorf("reading actor dir %s: %v", actorDir, err)
-	}
+	assertDirGone(t, "actor dir", ateletpath.ActorPath(actorUID))
+	assertDirGone(t, "actor snapshot dir", ateletpath.ActorSnapshotPath(actorUID))
 }
 
 // TestRestoreUsesRequestSandboxRuntime checks that Restore runs the actor with
@@ -519,13 +514,19 @@ func TestActivationFailureBeforeRegistration(t *testing.T) {
 		failure     string
 		wantErr     string
 		wantPresent bool
+		// publishedVolume leaves an external volume mount point with
+		// content in it, as a failed earlier attempt that published the
+		// volume would.
+		publishedVolume bool
 	}{
 		{name: "Run asset failure preserves projection", failure: "asset", wantErr: "asset unavailable", wantPresent: true},
-		{name: "Run partial reset removes registration", failure: "reset", wantErr: "while removing volume dir"},
+		{name: "Run partial reset removes registration", failure: "reset", wantErr: "while creating volumes dir"},
 		{name: "Run record failure removes registration", failure: "record", wantErr: "while recording sandbox assets"},
-		{name: "Restore partial reset removes registration", restore: true, failure: "reset", wantErr: "while removing volume dir"},
+		{name: "Restore partial reset removes registration", restore: true, failure: "reset", wantErr: "while creating volumes dir"},
 		{name: "Restore manifest failure removes registration", restore: true, failure: "manifest", wantErr: "no uploaded copy"},
 		{name: "Restore asset failure removes registration", restore: true, failure: "asset", wantErr: "asset unavailable"},
+		{name: "Run retry keeps a published volume", failure: "record", publishedVolume: true, wantErr: "while recording sandbox assets"},
+		{name: "Restore retry keeps a published volume", restore: true, failure: "manifest", publishedVolume: true, wantErr: "no uploaded copy"},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			useTempNodeDirs(t)
@@ -556,13 +557,9 @@ func TestActivationFailureBeforeRegistration(t *testing.T) {
 				}, map[string]string{"checkpoint.img": "guest-memory"})
 			}
 			if tc.failure == "reset" {
-				// Reset refuses to remove a populated external-volume directory,
-				// but has already removed the system-info roots by then.
-				dir := ateletpath.ExternalVolumeHostPath(actorUID, "mounted")
-				if err := os.MkdirAll(dir, 0o750); err != nil {
-					t.Fatal(err)
-				}
-				if err := os.WriteFile(filepath.Join(dir, "keep"), []byte("volume data"), 0o600); err != nil {
+				// A file where the volumes dir goes fails the reset after it
+				// has removed the system-info roots.
+				if err := os.WriteFile(ateletpath.ExternalVolumeMountsDir(actorUID), nil, 0o600); err != nil {
 					t.Fatal(err)
 				}
 			}
@@ -571,6 +568,18 @@ func TestActivationFailureBeforeRegistration(t *testing.T) {
 					t.Fatal(err)
 				}
 			}
+			// A non-empty dir fails os.Remove like a live mount (EBUSY)
+			// would, so the activation only gets past the reset to fail
+			// later with wantErr if the reset leaves volumes/ alone.
+			volumeFile := filepath.Join(ateletpath.ExternalVolumeHostPath(actorUID, "data"), "data.txt")
+			if tc.publishedVolume {
+				if err := os.MkdirAll(filepath.Dir(volumeFile), 0o755); err != nil {
+					t.Fatal(err)
+				}
+				if err := os.WriteFile(volumeFile, []byte("volume data"), 0o600); err != nil {
+					t.Fatal(err)
+				}
+			}
 			// The live activation's staged binary: a Run that fails before
 			// it swaps its own set in must leave it, and a Run that fails
 			// after must still leave nothing half-staged behind.
@@ -610,6 +619,11 @@ func TestActivationFailureBeforeRegistration(t *testing.T) {
 			if _, err := os.Lstat(ateletpath.SandboxAssetsStagingDir(actorUID)); !os.IsNotExist(err) {
 				t.Errorf("failed activation left the staging dir behind (lstat err = %v)", err)
 			}
+			if tc.publishedVolume {
+				if got, err := os.ReadFile(volumeFile); err != nil || string(got) != "volume data" {
+					t.Errorf("activation touched the published volume: %q, %v", got, err)
+				}
+			}
 
 			// Inspect disk and registry before changing the bundle hash: a later
 			// rotation can hide a stale registration by recreating deleted files.
@@ -1101,6 +1115,9 @@ func TestTerminateWithoutWorkerPodUID(t *testing.T) {
 	if _, err := os.Stat(ateletpath.LocalCheckpointsDir(actorUID)); !os.IsNotExist(err) {
 		t.Errorf("local checkpoint dir survived terminate: %v", err)
 	}
+	if _, err := os.Stat(ateletpath.ActorSnapshotPath(actorUID)); !os.IsNotExist(err) {
+		t.Errorf("actor snapshot dir survived terminate: %v", err)
+	}
 	if _, err := os.Stat(ateletpath.ActorPath(actorUID)); !os.IsNotExist(err) {
 		t.Errorf("actor dir survived terminate: %v", err)
 	}