Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions cmd/ateom-microvm/actorpath.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
//go:build linux

// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The paths ateom-microvm derives from the ActorDirs atelet sends. The
// directory under root_dir is ateom-microvm's own: atelet does not know it.

package main

import (
"path/filepath"

"github.com/agent-substrate/substrate/internal/proto/ateompb"
)

// ociBundlePath is the container's OCI bundle.
func ociBundlePath(actorDirs *ateompb.ActorDirs, containerName string) string {
return filepath.Join(actorDirs.GetOciBundleDir(), containerName)
}

// rootfsUpperDir is the host directory backing the actor's rootfs overlay
// uppers: one subdirectory per container (see kata.UpperWorkDirs). Local to
// this binary: atelet never touches it, so it is not one of the ActorDirs.
func rootfsUpperDir(actorDirs *ateompb.ActorDirs) string {
return filepath.Join(actorDirs.GetRootDir(), "rootfs-upper")
}
34 changes: 20 additions & 14 deletions cmd/ateom-microvm/checkpoint.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ import (

"github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/ch"
"github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata"
"github.com/agent-substrate/substrate/internal/ateompath"
"github.com/agent-substrate/substrate/internal/ateomstats"
"github.com/agent-substrate/substrate/internal/imagecache"
"github.com/agent-substrate/substrate/internal/proto/ateompb"
Expand All @@ -46,7 +45,7 @@ import (
// What the snapshot holds depends on the requested scope:
//
// - FULL: the whole guest. ateom drives the CH REST api-socket: pause -> snapshot
// file://<CheckpointStateDir> (config.json + state.json + sparse memory-ranges)
// file://<checkpoint_dir> (config.json + state.json + sparse memory-ranges)
// -> tear the VMM down. Each container's rootfs is overlay(virtio-fs RO lower +
// disk-backed upper): the upper is host-backed like the durable-dir volumes and
// ships alongside as its own tar (see rootfsupper.go); process memory persists
Expand All @@ -62,6 +61,9 @@ import (
// Allow checkpointing even if the pod is shutting down. This will allow actors
// (or the harness) to suspend on shutdown.
func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.CheckpointWorkloadRequest) (*ateompb.CheckpointWorkloadResponse, error) {
if err := validateActorDirs(req.GetActorDirs()); err != nil {
return nil, err
}
if !s.locks.Lock(ctx, req.GetActorUid()) {
return nil, status.Error(codes.Canceled, "gave up waiting for the actor's lock")
}
Expand All @@ -77,6 +79,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec
}

actorUID := req.GetActorUid()
actorDirs := req.GetActorDirs()

s.actorLogger.EmitLifecycleLog(ctx, "Actor checkpointing", attribution)

Expand Down Expand Up @@ -121,7 +124,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec
}
dPause := time.Since(tPause)

checkpointDir := ateompath.CheckpointStateDir(actorUID)
checkpointDir := actorDirs.GetCheckpointDir()
// Start from a clean dir so CH's snapshot files are the only contents.
if err := os.RemoveAll(checkpointDir); err != nil {
return nil, fmt.Errorf("while clearing checkpoint dir %q: %w", checkpointDir, err)
Expand Down Expand Up @@ -158,7 +161,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec
if durable {
g.Go(func() error {
t := time.Now()
if err := tarDurableVolumes(gctx, ateompath.DurableDirVolumeMountsDir(actorUID), checkpointDir); err != nil {
if err := tarDurableVolumes(gctx, actorDirs.GetDurableDirVolumeMountsDir(), checkpointDir); err != nil {
return err
}
dDurable = time.Since(t)
Expand All @@ -168,7 +171,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec
if scope == ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL {
g.Go(func() error {
t := time.Now()
if err := tarRootfsUpper(gctx, rootfsUpperDir(actorUID), checkpointDir); err != nil {
if err := tarRootfsUpper(gctx, rootfsUpperDir(actorDirs), checkpointDir); err != nil {
return err
}
dUpper = time.Since(t)
Expand All @@ -190,7 +193,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec
// Tear down: the actor returns to "available". Best-effort; the snapshot is
// already on disk for atelet to ship.
tTeardown := time.Now()
if err := s.terminateWorkload(ctx, attribution); err != nil {
if err := s.terminateWorkload(ctx, attribution, actorDirs); err != nil {
slog.WarnContext(ctx, "failed to terminate workload after checkpoint",
slog.String("actor", attribution.Ref.String()),
slog.String("actorUID", actorUID),
Expand Down Expand Up @@ -284,7 +287,7 @@ func listFiles(dir string) ([]string, error) {

// teardownActor stops the ateom-owned CH VMM for an actor. ra may be
// nil (e.g. ateom restarted and lost in-memory state).
func (s *AteomService) teardownActor(ctx context.Context, id string, ra *runningActor, client *ch.Client) error {
func (s *AteomService) teardownActor(ctx context.Context, id string, actorDirs *ateompb.ActorDirs, ra *runningActor, client *ch.Client) error {
// Stop offering the guest to GetWorkloadStats first, before anything below
// makes it stop answering. Clearing it here rather than alongside the
// attribution is what keeps a poll that lands mid-teardown on the
Expand Down Expand Up @@ -340,13 +343,13 @@ func (s *AteomService) teardownActor(ctx context.Context, id string, ra *running
// Remove the rootfs upper dir: ateom owns it — atelet's actor-dir reset
// doesn't know it — and its absence is what marks a worker as holding no
// disk-backed upper. Runs after the checkpoint tar, which is already on disk.
if err := os.RemoveAll(rootfsUpperDir(id)); err != nil {
if err := os.RemoveAll(rootfsUpperDir(actorDirs)); err != nil {
slog.WarnContext(ctx, "Failed to remove rootfs upper dir", slog.String("actorUID", id), slog.Any("err", err))
}

// Detach the bundle rootfs overlays composed in buildActorContainers, so
// atelet's bundle wipe doesn't strand live mounts in this namespace.
if err := imagecache.UnmountAllUnder(ateompath.OCIBundleDir(id)); err != nil {
if err := imagecache.UnmountAllUnder(actorDirs.GetOciBundleDir()); err != nil {
if !errors.Is(err, os.ErrNotExist) {
errs = append(errs, fmt.Errorf("while unmounting bundle rootfs overlays: %w", err))
}
Expand All @@ -357,14 +360,17 @@ func (s *AteomService) teardownActor(ctx context.Context, id string, ra *running
// TerminateWorkload stops the running actor, tears down its VMM, and cleans up
// networking and overlays.
func (s *AteomService) TerminateWorkload(ctx context.Context, req *ateompb.TerminateWorkloadRequest) (*ateompb.TerminateWorkloadResponse, error) {
if err := validateActorDirs(req.GetActorDirs()); err != nil {
return nil, err
}
if !s.locks.Lock(ctx, req.GetActorUid()) {
return nil, status.Error(codes.Canceled, "gave up waiting for the actor's lock")
}
defer s.locks.Unlock(req.GetActorUid())

attribution := ateomstats.ActorAttributionFromRequest(req)

if err := s.terminateWorkload(ctx, attribution); err != nil {
if err := s.terminateWorkload(ctx, attribution, req.GetActorDirs()); err != nil {
return nil, fmt.Errorf("failed to terminate workload: %w", err)
}

Expand All @@ -374,23 +380,23 @@ func (s *AteomService) TerminateWorkload(ctx context.Context, req *ateompb.Termi
}

// stopActorVM tears down the actor's micro-VM, if any, keeping it hosted.
func (s *AteomService) stopActorVM(ctx context.Context, actorUID string) error {
func (s *AteomService) stopActorVM(ctx context.Context, actorUID string, actorDirs *ateompb.ActorDirs) error {
ra := s.runningVM(actorUID)
chSocket := kata.CLHSocketPath(actorUID)
if ra != nil && ra.apiSocket != "" {
chSocket = ra.apiSocket
}
return s.teardownActor(ctx, actorUID, ra, ch.NewClient(chSocket))
return s.teardownActor(ctx, actorUID, actorDirs, ra, ch.NewClient(chSocket))
}

func (s *AteomService) terminateWorkload(ctx context.Context, actor resources.ActorAttribution) error {
func (s *AteomService) terminateWorkload(ctx context.Context, actor resources.ActorAttribution, actorDirs *ateompb.ActorDirs) error {
var errs []error
if err := s.deactivateActorNetworking(ctx, actor); err != nil {
errs = append(errs, fmt.Errorf("while deactivating actor networking: %w", err))
}

actorUID := actor.UID
if err := s.stopActorVM(ctx, actorUID); err != nil {
if err := s.stopActorVM(ctx, actorUID, actorDirs); err != nil {
errs = append(errs, fmt.Errorf("while tearing down actor: %w", err))
}
// Remove attribution after teardown; a failed checkpoint may leave the VM running.
Expand Down
6 changes: 2 additions & 4 deletions cmd/ateom-microvm/csi.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import (
"os"

"github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata"
"github.com/agent-substrate/substrate/internal/ateompath"
"github.com/agent-substrate/substrate/internal/ocispec"
"github.com/agent-substrate/substrate/internal/proto/ateompb"
)
Expand All @@ -37,10 +36,9 @@ func hasCsiVolumes(containers []*ateompb.Container) bool {
return false
}

// stageCsiVolumes bind-mounts the actor's host CSI volumes directory
// stageCsiVolumes bind-mounts src, the actor's host CSI volumes directory,
// into the sandbox's shared virtio-fs tree at SharedDir(actorUID)/csi.
func (s *AteomService) stageCsiVolumes(ctx context.Context, actorUID string) error {
src := ateompath.VolumesDir(actorUID)
func (s *AteomService) stageCsiVolumes(ctx context.Context, actorUID, src string) error {
if _, err := os.Stat(src); err != nil {
return fmt.Errorf("while checking CSI volumes dir %q: %w", src, err)
}
Expand Down
10 changes: 4 additions & 6 deletions cmd/ateom-microvm/durable.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ package main
// state: it survives suspend/resume and, under the Data snapshot scope, is the
// ONLY thing captured (the workload cold-starts on restore). The host side is
// owned by atelet, which creates one directory per volume under
// ateompath.DurableDirVolumeMountsDir(actorUID) and wipes them when the actor's
// ActorDirs.durable_dir_volume_mounts_dir and wipes them when the actor's
// directories are reset.
//
// ateom exposes that host directory to the guest under the single kataShared
Expand All @@ -42,7 +42,6 @@ import (
"path/filepath"

"github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata"
"github.com/agent-substrate/substrate/internal/ateompath"
"github.com/agent-substrate/substrate/internal/ocispec"
"github.com/agent-substrate/substrate/internal/proto/ateompb"
"github.com/agent-substrate/substrate/internal/resources"
Expand All @@ -51,7 +50,7 @@ import (

// durableTarFile is the snapshot file holding the tar of the actor's durable-dir
// volumes. Its entries are <volumeName>/... relative to
// ateompath.DurableDirVolumeMountsDir, so extraction restores the same layout.
// ActorDirs.durable_dir_volume_mounts_dir, so extraction restores the same layout.
// The name is shared with atelet, which uses it to carve durable data out of a
// FULL snapshot's file set when uploading a paused checkpoint as DATA.
const durableTarFile = resources.DurableDirTarFile
Expand All @@ -66,10 +65,9 @@ func hasDurableVolumes(containers []*ateompb.Container) bool {
return false
}

// stageDurableVolumes bind-mounts the actor's host durable-dir directory
// stageDurableVolumes bind-mounts src, the actor's host durable-dir directory,
// into the sandbox's shared virtio-fs tree at SharedDir(actorUID)/durable.
func (s *AteomService) stageDurableVolumes(ctx context.Context, actorUID string) error {
src := ateompath.DurableDirVolumeMountsDir(actorUID)
func (s *AteomService) stageDurableVolumes(ctx context.Context, actorUID, src string) error {
if _, err := os.Stat(src); err != nil {
return fmt.Errorf("while checking durable-dir volumes dir %q: %w", src, err)
}
Expand Down
9 changes: 9 additions & 0 deletions cmd/ateom-microvm/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ import (
"google.golang.org/grpc/codes"
"google.golang.org/grpc/reflection"
"google.golang.org/grpc/status"
"k8s.io/apimachinery/pkg/util/validation/field"
)

var (
Expand Down Expand Up @@ -551,6 +552,14 @@ func (s *AteomService) beginRPC(actorUID, name string, cancel context.CancelFunc
return release, nil
}

// validateActorDirs rejects a request whose actor directories are unusable.
func validateActorDirs(actorDirs *ateompb.ActorDirs) error {
if errs := resources.ValidateActorDirs(actorDirs, field.NewPath("actor_dirs")); len(errs) > 0 {
return status.Error(codes.InvalidArgument, errs.ToAggregate().Error())
}
return nil
}

// rejectIfDraining returns a codes.Unavailable error if ateom has begun graceful
// shutdown, so the control plane reschedules the actor onto a live worker.
func (s *AteomService) rejectIfDraining() error {
Expand Down
25 changes: 14 additions & 11 deletions cmd/ateom-microvm/restore.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ import (

"github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/ch"
"github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata"
"github.com/agent-substrate/substrate/internal/ateompath"
"github.com/agent-substrate/substrate/internal/imagecache"
"github.com/agent-substrate/substrate/internal/proto/ateompb"
"github.com/agent-substrate/substrate/internal/resources"
Expand Down Expand Up @@ -73,14 +72,17 @@ func restoreMemMode(ctx context.Context, info ch.VMMInfo) string {
// (restoreFullScope).
// - DATA: there is no guest to resume — re-materialize the durable-dir volumes and
// cold-boot the actor, which starts its containers afresh from the OCI image.
// - DATA_ON_GOLDEN: atelet staged a combined set into RestoreStateDir — the
// - DATA_ON_GOLDEN: atelet staged a combined set into restore_dir — the
// guest files (memory + VM state) from the template's golden snapshot plus
// the durable-dir tar from the actor's own snapshot — so this restores
// exactly like FULL: the golden guest resumes over the actor's data.
//
// Contract with atelet: the snapshot's files have been downloaded to RestoreStateDir,
// and the durable-dir volume directories re-created (empty).
// Contract with atelet: the snapshot's files have been downloaded to
// ActorDirs.restore_dir, and the durable-dir volume directories re-created (empty).
func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.RestoreWorkloadRequest) (resp *ateompb.RestoreWorkloadResponse, retErr error) {
if err := validateActorDirs(req.GetActorDirs()); err != nil {
return nil, err
}
if !s.locks.Lock(ctx, req.GetActorUid()) {
return nil, status.Error(codes.Canceled, "gave up waiting for the actor's lock")
}
Expand All @@ -102,15 +104,16 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore
p := actorBootParams{
actorRef: resources.ActorRef{Atespace: req.GetAtespace(), Name: req.GetActorName()},
actorUID: req.GetActorUid(),
actorDirs: req.GetActorDirs(),
templateAtespace: req.GetActorTemplateAtespace(),
templateName: req.GetActorTemplateName(),
containers: req.GetSpec().GetContainers(),
assetPaths: req.GetRuntimeAssetPaths(),
egressGateway: req.GetEgressGateway(),
size: sizing.FromLimits(req.GetCpuMilli(), req.GetMemoryBytes()),
}
restoreDir := ateompath.RestoreStateDir(p.actorUID)
durableDir := ateompath.DurableDirVolumeMountsDir(p.actorUID)
restoreDir := p.actorDirs.GetRestoreDir()
durableDir := p.actorDirs.GetDurableDirVolumeMountsDir()
tStart := time.Now()

attribution := p.actorAttribution()
Expand All @@ -119,7 +122,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore
// A VM still running for this actor would be dropped from tracking by the
// re-host below and left running, so stop it first.
if s.runningVM(attribution.UID) != nil {
if err := s.stopActorVM(ctx, attribution.UID); err != nil {
if err := s.stopActorVM(ctx, attribution.UID, req.GetActorDirs()); err != nil {
return nil, fmt.Errorf("while stopping the actor's previous micro-VM: %w", err)
}
}
Expand Down Expand Up @@ -225,7 +228,7 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams,
untarDone := make(chan error, 1)
untarJoined := false
go func() {
untarDone <- untarRootfsUpper(rootfsUpperDir(actorUID), restoreDir)
untarDone <- untarRootfsUpper(rootfsUpperDir(p.actorDirs), restoreDir)
}()
defer func() {
if !untarJoined {
Expand All @@ -247,7 +250,7 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams,
if len(containers) > maxActorContainers {
return status.Errorf(codes.Unimplemented, "ateom-microvm supports at most %d containers, got %d", maxActorContainers, len(containers))
}
ctrs, err := s.buildActorContainers(actorUID, containers)
ctrs, err := s.buildActorContainers(p.actorDirs, containers)
if err != nil {
return err
}
Expand All @@ -266,7 +269,7 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams,
return err
}
defer leaf.Close()
vfsdCmd, err := s.stageMergedRootfs(ctx, rr, actorUID, ctrs, containers, leaf.SysProcAttr())
vfsdCmd, err := s.stageMergedRootfs(ctx, rr, actorUID, p.actorDirs, ctrs, containers, leaf.SysProcAttr())
if err != nil {
return err
}
Expand All @@ -293,7 +296,7 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams,
}
// Detach any bundle rootfs overlays mounted by buildActorContainers
// before the failure, mirroring teardownActor's cleanup.
if err := imagecache.UnmountAllUnder(ateompath.OCIBundleDir(actorUID)); err != nil {
if err := imagecache.UnmountAllUnder(p.actorDirs.GetOciBundleDir()); err != nil {
slog.WarnContext(ctx, "Failed to unmount bundle rootfs overlays after Restore failure", slog.Any("err", err))
}
}
Expand Down
13 changes: 3 additions & 10 deletions cmd/ateom-microvm/rootfsupper.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ import (
"path/filepath"
"strings"

"github.com/agent-substrate/substrate/internal/ateompath"
"github.com/agent-substrate/substrate/internal/proto/ateompb"
"github.com/agent-substrate/substrate/internal/tarutil"
)

Expand All @@ -62,20 +62,13 @@ import (
// tarRootfsUpper); StageMergedRootfs recreates it at mount.
const rootfsUpperTarFile = "rootfs-upper.tar"

// rootfsUpperDir is the host directory backing the actor's rootfs overlay
// uppers: one subdirectory per container (see kata.UpperWorkDirs). Local to
// this binary — no other component touches it — hence not in ateompath.
func rootfsUpperDir(actorUID string) string {
return filepath.Join(ateompath.ActorPath(actorUID), "rootfs-upper")
}

// resetRootfsUpperDir gives a cold boot a pristine upper directory: a cold
// boot must start from the bare image, and atelet's actor-dir reset does not
// know about this directory, so ateom wipes any previous activation's contents
// itself. The per-container fs/work subdirectories are created by the overlay
// staging (kata.StageMergedRootfs).
func resetRootfsUpperDir(actorUID string) error {
dir := rootfsUpperDir(actorUID)
func resetRootfsUpperDir(actorDirs *ateompb.ActorDirs) error {
dir := rootfsUpperDir(actorDirs)
if err := os.RemoveAll(dir); err != nil {
return fmt.Errorf("while clearing rootfs upper dir %q: %w", dir, err)
}
Expand Down
Loading
Loading