diff --git a/cmd/atelet/internal/ateletpath/ateletpath.go b/cmd/atelet/internal/ateletpath/ateletpath.go index deefce445c..9c2a9ebef8 100644 --- a/cmd/atelet/internal/ateletpath/ateletpath.go +++ b/cmd/atelet/internal/ateletpath/ateletpath.go @@ -64,13 +64,6 @@ func ActorSandboxAssetsFile(actorUID string) string { ) } -func RunSCStateDir(actorUID string) string { - return filepath.Join( - ActorPath(actorUID), - "runsc-state", - ) -} - func OCIBundleDir(actorUID string) string { return filepath.Join( ActorPath(actorUID), @@ -171,13 +164,6 @@ func RestoreStateDir(actorUID string) string { ) } -func PIDFileDir(actorUID string) string { - return filepath.Join( - ActorPath(actorUID), - "pidfiles", - ) -} - func VolumesDir(actorUID string) string { return filepath.Join( ActorPath(actorUID), diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 6c29011834..7176ce4f17 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -1874,22 +1874,6 @@ func resetActorDirs(actorUID string) error { return wrapFileSystemErr("while creating bundle dir: %w", err) } - runscDir := ateletpath.RunSCStateDir(actorUID) - if err := os.RemoveAll(runscDir); err != nil { - return wrapFileSystemErr("while deleting runsc state dir: %w", err) - } - if err := os.MkdirAll(runscDir, 0o700); err != nil { - return wrapFileSystemErr("while creating runsc state dir: %w", err) - } - - pidFileDir := ateletpath.PIDFileDir(actorUID) - if err := os.RemoveAll(pidFileDir); err != nil { - return wrapFileSystemErr("while deleting PID file dir: %w", err) - } - if err := os.MkdirAll(pidFileDir, 0o700); err != nil { - return wrapFileSystemErr("while creating PID file dir: %w", err) - } - checkpointDir := ateletpath.CheckpointStateDir(actorUID) if err := os.RemoveAll(checkpointDir); err != nil { return wrapFileSystemErr("while deleting checkpoint-state dir: %w", err) diff --git a/cmd/ateom-gvisor/actorpath.go b/cmd/ateom-gvisor/actorpath.go new file mode 100644 index 0000000000..b2855a9a13 --- /dev/null +++ b/cmd/ateom-gvisor/actorpath.go @@ -0,0 +1,71 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The paths ateom-gvisor derives from the ActorDirs atelet sends. + +package main + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/agent-substrate/substrate/internal/imagecache" + "github.com/agent-substrate/substrate/internal/proto/ateompb" +) + +// ociBundlePath is the container's OCI bundle. +func ociBundlePath(actorDirs *ateompb.ActorDirs, containerName string) string { + return filepath.Join(actorDirs.GetOciBundleDir(), containerName) +} + +// runscStateDir is runsc --root for the actor's sandbox. +func runscStateDir(actorDirs *ateompb.ActorDirs) string { + return filepath.Join(actorDirs.GetRootDir(), "runsc-state") +} + +// pidFileDir is where runsc writes .pid. +func pidFileDir(actorDirs *ateompb.ActorDirs) string { + return filepath.Join(actorDirs.GetRootDir(), "pidfiles") +} + +// pidFilePath is where runsc writes the container's pid. +func pidFilePath(actorDirs *ateompb.ActorDirs, containerName string) string { + return filepath.Join(pidFileDir(actorDirs), containerName+".pid") +} + +// resolvConfPath is the resolver bind source outside the actor's rootfs. +func resolvConfPath(actorDirs *ateompb.ActorDirs) string { + return filepath.Join(actorDirs.GetRootDir(), "resolv.conf") +} + +// resetRunscStateAndPidFileDirs empties both directories for a new activation. +// runsc can leave mounts behind in its state directory (its null-netns), which +// must be detached in this mount namespace before they can be removed. +func resetRunscStateAndPidFileDirs(actorDirs *ateompb.ActorDirs) error { + for _, dir := range []string{runscStateDir(actorDirs), pidFileDir(actorDirs)} { + if err := imagecache.UnmountAllUnder(dir); err != nil { + return fmt.Errorf("while unmounting under %q: %w", dir, err) + } + if err := os.RemoveAll(dir); err != nil { + return fmt.Errorf("while clearing %q: %w", dir, err) + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return fmt.Errorf("while creating %q: %w", dir, err) + } + } + return nil +} diff --git a/cmd/ateom-gvisor/actorpath_test.go b/cmd/ateom-gvisor/actorpath_test.go new file mode 100644 index 0000000000..8d24baf39a --- /dev/null +++ b/cmd/ateom-gvisor/actorpath_test.go @@ -0,0 +1,71 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "path/filepath" + "testing" + + "golang.org/x/sys/unix" + + "github.com/agent-substrate/substrate/internal/proto/ateompb" + "github.com/agent-substrate/substrate/internal/roottest" +) + +func TestResetRunscStateAndPidFileDirs(t *testing.T) { + actorDirs := &ateompb.ActorDirs{RootDir: t.TempDir()} + if err := os.MkdirAll(filepath.Join(runscStateDir(actorDirs), "stale"), 0o700); err != nil { + t.Fatal(err) + } + if err := resetRunscStateAndPidFileDirs(actorDirs); err != nil { + t.Fatalf("resetRunscStateAndPidFileDirs() = %v", err) + } + for _, dir := range []string{runscStateDir(actorDirs), pidFileDir(actorDirs)} { + if entries, err := os.ReadDir(dir); err != nil || len(entries) != 0 { + t.Errorf("ReadDir(%q) = %v, %v; want an empty directory", dir, entries, err) + } + } +} + +// runsc leaves mounts in its state directory; unlinking one fails with EBUSY +// until it is detached. +func TestResetRunscStateAndPidFileDirs_DetachesMounts(t *testing.T) { + roottest.Require(t, "mount/unmount") + actorDirs := &ateompb.ActorDirs{RootDir: t.TempDir()} + if err := os.MkdirAll(runscStateDir(actorDirs), 0o700); err != nil { + t.Fatal(err) + } + src := filepath.Join(t.TempDir(), "netns") + target := filepath.Join(runscStateDir(actorDirs), "null-netns") + for _, f := range []string{src, target} { + if err := os.WriteFile(f, nil, 0o600); err != nil { + t.Fatal(err) + } + } + if err := unix.Mount(src, target, "", unix.MS_BIND, ""); err != nil { + t.Fatalf("bind mount: %v", err) + } + t.Cleanup(func() { _ = unix.Unmount(target, unix.MNT_DETACH) }) + + if err := resetRunscStateAndPidFileDirs(actorDirs); err != nil { + t.Fatalf("resetRunscStateAndPidFileDirs() = %v", err) + } + if entries, err := os.ReadDir(runscStateDir(actorDirs)); err != nil || len(entries) != 0 { + t.Errorf("ReadDir(%q) = %v, %v; want an empty directory", runscStateDir(actorDirs), entries, err) + } +} diff --git a/cmd/ateom-gvisor/durable.go b/cmd/ateom-gvisor/durable.go index 65775e6bd9..b58ab29169 100644 --- a/cmd/ateom-gvisor/durable.go +++ b/cmd/ateom-gvisor/durable.go @@ -30,7 +30,7 @@ import ( // durableTarFile is the snapshot file holding the tar of the actor's durable-dir // volumes. Its entries are /... relative to -// ateompath.DurableDirVolumeMountsDir, so extraction restores the same layout. +// ActorDirs.durable_dir_volume_mounts_dir, so extraction restores the same layout. // The name is shared with atelet, which uses it to carve durable data out of a // FULL snapshot's file set when uploading a paused checkpoint as DATA. const durableTarFile = resources.DurableDirTarFile diff --git a/cmd/ateom-gvisor/hosted.go b/cmd/ateom-gvisor/hosted.go index 6c358c196b..aa32e87663 100644 --- a/cmd/ateom-gvisor/hosted.go +++ b/cmd/ateom-gvisor/hosted.go @@ -27,6 +27,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateomnet" "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/resources" ) @@ -35,6 +36,8 @@ type hostedActor struct { // Attribution is available during boot. attribution resources.ActorAttribution network *ateomnet.SandboxSession + // resolvConf is the actor's resolver bind source, removed with its network. + resolvConf string // Set once the containers exist. session *workloadSession } @@ -56,7 +59,7 @@ func (s *AteomService) admitActor(attribution resources.ActorAttribution) (*host } // hostActor sets up the actor's network, replacing any stale one. -func (s *AteomService) hostActor(ctx context.Context, attribution resources.ActorAttribution) (*hostedActor, error) { +func (s *AteomService) hostActor(ctx context.Context, attribution resources.ActorAttribution, actorDirs *ateompb.ActorDirs) (*hostedActor, error) { uid := attribution.UID if uid == "" { return nil, fmt.Errorf("actor UID is required") @@ -89,7 +92,8 @@ func (s *AteomService) hostActor(ctx context.Context, attribution resources.Acto } // Use the actor's gateway as its DNS resolver. - if _, err := actorResolvConf(uid); err != nil { + resolvConf, err := actorResolvConf(actorDirs) + if err != nil { _ = session.Close(ctx) s.actorsMu.Lock() delete(s.actors, uid) @@ -99,6 +103,7 @@ func (s *AteomService) hostActor(ctx context.Context, attribution resources.Acto s.actorsMu.Lock() hosted.network = session + hosted.resolvConf = resolvConf s.actorsMu.Unlock() return hosted, nil } @@ -122,7 +127,7 @@ func (s *AteomService) unhostActor(ctx context.Context, actorUID string) error { s.actorsMu.Unlock() }() - removeActorResolvConf(ctx, actorUID) + removeActorResolvConf(ctx, hosted.resolvConf) if hosted.network == nil { return nil } diff --git a/cmd/ateom-gvisor/hosted_test.go b/cmd/ateom-gvisor/hosted_test.go index a500738f73..b671ef508b 100644 --- a/cmd/ateom-gvisor/hosted_test.go +++ b/cmd/ateom-gvisor/hosted_test.go @@ -85,7 +85,7 @@ func TestDrainingActorsStillCountAgainstTheCeiling(t *testing.T) { maxActors: 1, draining: 1, } - if _, err := s.hostActor(context.Background(), resources.ActorAttribution{UID: "actor-a"}); err == nil { + if _, err := s.hostActor(context.Background(), resources.ActorAttribution{UID: "actor-a"}, nil); err == nil { t.Error("admitted an actor while a draining one still held the only place") } } diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 4f0e72aee1..a976fd8e12 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -40,7 +40,6 @@ import ( "github.com/agent-substrate/substrate/internal/ateomcapacity" "github.com/agent-substrate/substrate/internal/ateomcgroup" "github.com/agent-substrate/substrate/internal/ateomnet" - "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/ateomstats" "github.com/agent-substrate/substrate/internal/atunnel" "github.com/agent-substrate/substrate/internal/childreap" @@ -62,6 +61,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/reflection" "google.golang.org/grpc/status" + "k8s.io/apimachinery/pkg/util/validation/field" ) var ( @@ -602,7 +602,18 @@ func containerNames(containers []*ateompb.Container) []string { return names } +// validateActorDirs rejects a request whose actor directories are unusable. +func validateActorDirs(actorDirs *ateompb.ActorDirs) error { + if errs := resources.ValidateActorDirs(actorDirs, field.NewPath("actor_dirs")); len(errs) > 0 { + return status.Error(codes.InvalidArgument, errs.ToAggregate().Error()) + } + return nil +} + func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkloadRequest) (resp *ateompb.RunWorkloadResponse, retErr error) { + if err := validateActorDirs(req.GetActorDirs()); err != nil { + return nil, err + } if !s.locks.Lock(ctx, req.GetActorUid()) { return nil, status.Error(codes.Canceled, "gave up waiting for the actor's lock") } @@ -614,6 +625,9 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload return nil, err } defer release() + if err := resetRunscStateAndPidFileDirs(req.GetActorDirs()); err != nil { + return nil, err + } if err := s.deactivateActorNetworking(ctx, ateomstats.ActorAttributionFromRequest(req)); err != nil { return nil, err @@ -632,12 +646,13 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload return nil, err } // Publish attribution before boot so stats can include startup usage. - if _, err := s.hostActor(ctx, attribution); err != nil { + if _, err := s.hostActor(ctx, attribution, req.GetActorDirs()); err != nil { return nil, err } rcmd := &runsc{ path: req.GetRunscPath(), actorUID: req.GetActorUid(), + actorDirs: req.GetActorDirs(), size: sizing.FromLimits(req.GetCpuMilli(), req.GetMemoryBytes()), durableVolumes: durableVolumeNames(req.GetSpec()), } @@ -654,7 +669,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload // mounted, mirroring the post-checkpoint cleanup — otherwise they // linger in this namespace until atelet wipes the bundle dirs. // Run before the network cleanup. - if err := imagecache.UnmountAllUnder(ateompath.OCIBundleDir(req.GetActorUid())); err != nil { + if err := imagecache.UnmountAllUnder(req.GetActorDirs().GetOciBundleDir()); err != nil { slog.WarnContext(ctx, "Failed to unmount bundle rootfs overlays after Run failure", "actorUID", req.GetActorUid(), "err", err) } @@ -668,7 +683,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload // upper — because mounting is ateom's job (atelet runs with no // capabilities); runsc's gofer resolves the mount in this pod's mount // namespace. - if err := imagecache.SetupBundleRootfs(ateompath.OCIBundlePath(req.GetActorUid(), ocispec.PauseContainer)); err != nil { + if err := imagecache.SetupBundleRootfs(ociBundlePath(req.GetActorDirs(), ocispec.PauseContainer)); err != nil { return nil, fmt.Errorf("while composing pause rootfs: %w", err) } containersToDelete = append(containersToDelete, ocispec.PauseContainer) @@ -687,7 +702,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload return nil, fmt.Errorf("while starting json log pipe for %q: %w", ac.GetName(), err) } defer pw.Close() - if err := imagecache.SetupBundleRootfs(ateompath.OCIBundlePath(req.GetActorUid(), ac.GetName())); err != nil { + if err := imagecache.SetupBundleRootfs(ociBundlePath(req.GetActorDirs(), ac.GetName())); err != nil { return nil, fmt.Errorf("while composing %q rootfs: %w", ac.GetName(), err) } containersToDelete = append(containersToDelete, ac.GetName()) @@ -716,6 +731,9 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload // Allow checkpointing even if the pod is shutting down. This will allow actors // (or the harness) to suspend on shutdown. func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.CheckpointWorkloadRequest) (*ateompb.CheckpointWorkloadResponse, error) { + if err := validateActorDirs(req.GetActorDirs()); err != nil { + return nil, err + } if !s.locks.Lock(ctx, req.GetActorUid()) { return nil, status.Error(codes.Canceled, "gave up waiting for the actor's lock") } @@ -740,11 +758,12 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec // Checkpoint only saves state; no sizing is applied, so size is left zero. rcmd := &runsc{ - path: req.GetRunscPath(), - actorUID: req.GetActorUid(), + path: req.GetRunscPath(), + actorUID: req.GetActorUid(), + actorDirs: req.GetActorDirs(), } - checkpointPath := ateompath.CheckpointStateDir(req.GetActorUid()) + checkpointPath := req.GetActorDirs().GetCheckpointDir() if err := os.MkdirAll(checkpointPath, 0o700); err != nil { return nil, fmt.Errorf("while creating checkpoint directory: %w", err) } @@ -759,7 +778,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec if err := rcmd.cmdPause(ctx, ocispec.PauseContainer); err != nil { return nil, fmt.Errorf("while pausing pause container: %w", err) } - tarErr := tarDurableVolumes(ctx, ateompath.DurableDirVolumeMountsDir(req.GetActorUid()), checkpointPath) + tarErr := tarDurableVolumes(ctx, req.GetActorDirs().GetDurableDirVolumeMountsDir(), checkpointPath) // Undoing our own pause must not depend on the caller's context: // tarutil does not check ctx, so a deadline expiring mid-tar would // fail the resume instantly and leave the sandbox paused forever. @@ -778,7 +797,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec return nil, fmt.Errorf("while checkpointing pause: %w", err) } if hasDurableVolumes(req.GetSpec().GetContainers()) { - if err := tarDurableVolumes(ctx, ateompath.DurableDirVolumeMountsDir(req.GetActorUid()), checkpointPath); err != nil { + if err := tarDurableVolumes(ctx, req.GetActorDirs().GetDurableDirVolumeMountsDir(), checkpointPath); err != nil { return nil, fmt.Errorf("while archiving durable-dir volumes: %w", err) } } @@ -789,7 +808,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec // Cleanup the containers after checkpointing. This also unhosts the actor, // before the snapshot listing below can fail. // This is best-effort cleanup for actor containers that may have been left behind after checkpointing. - if err := s.terminateWorkload(ctx, attribution.Ref, attribution.UID, req.GetRunscPath(), req.GetSpec().GetContainers()); err != nil { + if err := s.terminateWorkload(ctx, attribution.Ref, attribution.UID, req.GetRunscPath(), req.GetActorDirs(), req.GetSpec().GetContainers()); err != nil { slog.WarnContext(ctx, "failed to terminate workload after checkpoint", slog.String("actor", attribution.Ref.String()), slog.String("actorUID", attribution.UID), @@ -883,6 +902,9 @@ func isContainerAlreadyGone(ctx context.Context, rcmd containerRuntime, name str } func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.RestoreWorkloadRequest) (resp *ateompb.RestoreWorkloadResponse, retErr error) { + if err := validateActorDirs(req.GetActorDirs()); err != nil { + return nil, err + } if !s.locks.Lock(ctx, req.GetActorUid()) { return nil, status.Error(codes.Canceled, "gave up waiting for the actor's lock") } @@ -894,6 +916,9 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore return nil, err } defer release() + if err := resetRunscStateAndPidFileDirs(req.GetActorDirs()); err != nil { + return nil, err + } if err := s.deactivateActorNetworking(ctx, ateomstats.ActorAttributionFromRequest(req)); err != nil { return nil, err @@ -912,12 +937,13 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore if err != nil { return nil, err } - if _, err := s.hostActor(ctx, attribution); err != nil { + if _, err := s.hostActor(ctx, attribution, req.GetActorDirs()); err != nil { return nil, err } rcmd := &runsc{ path: req.GetRunscPath(), actorUID: req.GetActorUid(), + actorDirs: req.GetActorDirs(), size: sizing.FromLimits(req.GetCpuMilli(), req.GetMemoryBytes()), durableVolumes: durableVolumeNames(req.GetSpec()), } @@ -931,7 +957,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore } deleteContainers(cleanupCtx, rcmd, containersToDelete, "Restore") // Same overlay detach as the Run-failure path above. - if err := imagecache.UnmountAllUnder(ateompath.OCIBundleDir(req.GetActorUid())); err != nil { + if err := imagecache.UnmountAllUnder(req.GetActorDirs().GetOciBundleDir()); err != nil { slog.WarnContext(ctx, "Failed to unmount bundle rootfs overlays after Restore failure", "actorUID", req.GetActorUid(), "err", err) } @@ -940,17 +966,17 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore } } }() - checkpointDir := ateompath.RestoreStateDir(req.GetActorUid()) + checkpointDir := req.GetActorDirs().GetRestoreDir() if hasDurableVolumes(req.GetSpec().GetContainers()) { - if err := untarDurableVolumes(ateompath.DurableDirVolumeMountsDir(req.GetActorUid()), checkpointDir); err != nil { + if err := untarDurableVolumes(req.GetActorDirs().GetDurableDirVolumeMountsDir(), checkpointDir); err != nil { return nil, fmt.Errorf("while restoring durable-dir volumes: %w", err) } } // Compose the pause rootfs before create (see RunWorkload). runsc restore // only needs the rootfs to hold the correct content; whether it came from // an untar or an overlay of cached layers is transparent to it. - if err := imagecache.SetupBundleRootfs(ateompath.OCIBundlePath(req.GetActorUid(), ocispec.PauseContainer)); err != nil { + if err := imagecache.SetupBundleRootfs(ociBundlePath(req.GetActorDirs(), ocispec.PauseContainer)); err != nil { return nil, fmt.Errorf("while composing pause rootfs: %w", err) } @@ -985,7 +1011,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore return nil, fmt.Errorf("while starting json log pipe for %q: %w", ac.GetName(), err) } defer pw.Close() - if err := imagecache.SetupBundleRootfs(ateompath.OCIBundlePath(req.GetActorUid(), ac.GetName())); err != nil { + if err := imagecache.SetupBundleRootfs(ociBundlePath(req.GetActorDirs(), ac.GetName())); err != nil { return nil, fmt.Errorf("while composing %q rootfs: %w", ac.GetName(), err) } switch req.GetScope() { @@ -1074,6 +1100,9 @@ func (s *AteomService) prepareActorEgress(ctx context.Context, actorAtespace, ac } func (s *AteomService) TerminateWorkload(ctx context.Context, req *ateompb.TerminateWorkloadRequest) (*ateompb.TerminateWorkloadResponse, error) { + if err := validateActorDirs(req.GetActorDirs()); err != nil { + return nil, err + } if !s.locks.Lock(ctx, req.GetActorUid()) { return nil, status.Error(codes.Canceled, "gave up waiting for the actor's lock") } @@ -1081,7 +1110,7 @@ func (s *AteomService) TerminateWorkload(ctx context.Context, req *ateompb.Termi attribution := ateomstats.ActorAttributionFromRequest(req) - if err := s.terminateWorkload(ctx, attribution.Ref, attribution.UID, req.GetRunscPath(), req.GetSpec().GetContainers()); err != nil { + if err := s.terminateWorkload(ctx, attribution.Ref, attribution.UID, req.GetRunscPath(), req.GetActorDirs(), req.GetSpec().GetContainers()); err != nil { return nil, fmt.Errorf("failed to terminate workload: %w", err) } @@ -1090,15 +1119,16 @@ func (s *AteomService) TerminateWorkload(ctx context.Context, req *ateompb.Termi return &ateompb.TerminateWorkloadResponse{}, nil } -func (s *AteomService) terminateWorkload(ctx context.Context, actorRef resources.ActorRef, actorUID, runscPath string, containers []*ateompb.Container) error { +func (s *AteomService) terminateWorkload(ctx context.Context, actorRef resources.ActorRef, actorUID, runscPath string, actorDirs *ateompb.ActorDirs, containers []*ateompb.Container) error { var errs []error if err := s.deactivateActorNetworking(ctx, resources.ActorAttribution{Ref: actorRef, UID: actorUID}); err != nil { errs = append(errs, fmt.Errorf("while deactivating actor networking: %w", err)) } rcmd := &runsc{ - path: runscPath, - actorUID: actorUID, + path: runscPath, + actorUID: actorUID, + actorDirs: actorDirs, } // Detached from the caller: a deadline mid-`runsc delete` would leave the @@ -1109,18 +1139,23 @@ func (s *AteomService) terminateWorkload(ctx context.Context, actorRef resources // with no bundle on disk. Best-effort: if they are already stopped, the // delete succeeds anyway. stopContainers(cleanupCtx, rcmd, containers) - // Keep this as best-effort cleanup: - // atelet resets the actor runsc, bundle, pidfile, and checkpoint + // Keep this as best-effort cleanup: atelet resets the bundle and checkpoint // directories after uploading the snapshot. if err := cleanupContainers(cleanupCtx, rcmd, containers); err != nil { errs = append(errs, fmt.Errorf("while cleaning up runsc containers: %w", err)) } + // The actor may resume on another worker, so this one may never see another + // Run or Restore for it. Reset files here to close the loop. + if err := resetRunscStateAndPidFileDirs(actorDirs); err != nil { + errs = append(errs, fmt.Errorf("while resetting runsc state and pid file dirs: %w", err)) + } + // Detach the overlay rootfs mounts before atelet wipes the bundle dirs // (deleting a bundle out from under a live mount in this namespace would // leave the mount orphaned until the pod restarts). Best-effort, same as // the container cleanup above. - if err := imagecache.UnmountAllUnder(ateompath.OCIBundleDir(actorUID)); err != nil { + if err := imagecache.UnmountAllUnder(actorDirs.GetOciBundleDir()); err != nil { errs = append(errs, fmt.Errorf("while unmounting bundle rootfs overlays: %w", err)) } diff --git a/cmd/ateom-gvisor/main_test.go b/cmd/ateom-gvisor/main_test.go new file mode 100644 index 0000000000..dd5b658838 --- /dev/null +++ b/cmd/ateom-gvisor/main_test.go @@ -0,0 +1,55 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "testing" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/agent-substrate/substrate/internal/proto/ateompb" +) + +// Every RPC rejects a request without ActorDirs before touching any state. +func TestRPCsRejectMissingActorDirs(t *testing.T) { + s := &AteomService{} + ctx := context.Background() + for name, call := range map[string]func() error{ + "RunWorkload": func() error { + _, err := s.RunWorkload(ctx, &ateompb.RunWorkloadRequest{}) + return err + }, + "RestoreWorkload": func() error { + _, err := s.RestoreWorkload(ctx, &ateompb.RestoreWorkloadRequest{}) + return err + }, + "CheckpointWorkload": func() error { + _, err := s.CheckpointWorkload(ctx, &ateompb.CheckpointWorkloadRequest{}) + return err + }, + "TerminateWorkload": func() error { + _, err := s.TerminateWorkload(ctx, &ateompb.TerminateWorkloadRequest{}) + return err + }, + } { + if got := status.Code(call()); got != codes.InvalidArgument { + t.Errorf("%s() code = %v, want %v", name, got, codes.InvalidArgument) + } + } +} diff --git a/cmd/ateom-gvisor/runsc.go b/cmd/ateom-gvisor/runsc.go index bbf9e13ed5..dbe62a134d 100644 --- a/cmd/ateom-gvisor/runsc.go +++ b/cmd/ateom-gvisor/runsc.go @@ -29,7 +29,6 @@ import ( "strings" "syscall" - "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/ocispec" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/sizing" @@ -38,6 +37,8 @@ import ( type runsc struct { path string actorUID string + // actorDirs are the actor's directories, from the request. + actorDirs *ateompb.ActorDirs // size is the actor's declared limits. size sizing.SandboxSize // durableVolumes are the durable-dir volume names declared to the sandbox. @@ -59,7 +60,7 @@ func durableVolumeNames(spec *ateompb.WorkloadSpec) []string { // shapeSpec loads, shapes for gVisor, and saves the container's OCI spec. func (r *runsc) shapeSpec(containerName string) error { - bundle := ateompath.OCIBundlePath(r.actorUID, containerName) + bundle := ociBundlePath(r.actorDirs, containerName) spec, err := ocispec.Load(bundle) if err != nil { return err @@ -69,7 +70,7 @@ func (r *runsc) shapeSpec(containerName string) error { ContainerName: containerName, DurableVolumes: r.durableVolumes, Size: r.size, - ResolvConf: ateompath.ActorResolvConfPath(r.actorUID), + ResolvConf: resolvConfPath(r.actorDirs), }) return ocispec.Save(bundle, spec) } @@ -85,11 +86,11 @@ func (r *runsc) cmdCreate(ctx context.Context, out io.Writer, containerName stri "-log-format", "json", "--alsologtostderr", // "-debug", - // "-debug-log", ateompath.RunscDebugLogDir(r.actorUID, containerName) + "/", + // "-debug-log", filepath.Join(r.actorDirs.GetRootDir(), "runsc-debug-logs", containerName) + "/", // "-debug-to-user-log", // "-log-packets", // "-strace", - "-root", ateompath.RunSCStateDir(r.actorUID), + "-root", runscStateDir(r.actorDirs), // Provision the sentry's vCPU count from the cgroup CPU quota written by // sizing.ApplyToOCISpec, so the sandbox is sized to the pod's limit (runsc // otherwise sizes to all host CPUs). Global flag: before the subcommand. @@ -97,8 +98,8 @@ func (r *runsc) cmdCreate(ctx context.Context, out io.Writer, containerName stri } args = append(args, "create", - "-bundle", ateompath.OCIBundlePath(r.actorUID, containerName), - "-pid-file", ateompath.PIDFilePath(r.actorUID, containerName), + "-bundle", ociBundlePath(r.actorDirs, containerName), + "-pid-file", pidFilePath(r.actorDirs, containerName), ) args = append(args, additionalArgs...) @@ -126,12 +127,12 @@ func (r *runsc) cmdStart(ctx context.Context, out io.Writer, containerName strin "-log-format", "json", "--alsologtostderr", // "-debug", - // "-debug-log", ateompath.RunscDebugLogDir(r.actorUID, containerName)+"/", + // "-debug-log", filepath.Join(r.actorDirs.GetRootDir(), "runsc-debug-logs", containerName) + "/", // "-debug-to-user-log", // "-log-packets", // "-strace", "-allow-connected-on-save", - "-root", ateompath.RunSCStateDir(r.actorUID), + "-root", runscStateDir(r.actorDirs), } startArgs = append(startArgs, "start", containerName) cmd := exec.CommandContext(ctx, r.path, startArgs...) @@ -155,11 +156,11 @@ func (r *runsc) cmdCheckpoint(ctx context.Context, containerName, checkpointPath "-log-format", "json", "--alsologtostderr", // "-debug", - // "-debug-log", ateompath.RunscDebugLogDir(r.actorUID, containerName)+"/", + // "-debug-log", filepath.Join(r.actorDirs.GetRootDir(), "runsc-debug-logs", containerName) + "/", // "-debug-to-user-log", // "-log-packets", // "-strace", - "-root", ateompath.RunSCStateDir(r.actorUID), + "-root", runscStateDir(r.actorDirs), "checkpoint", "-image-path", checkpointPath, containerName, // Name of the container @@ -181,11 +182,11 @@ func (r *runsc) cmdFsCheckpoint(ctx context.Context, containerName, checkpointPa "-log-format", "json", "--alsologtostderr", // "-debug", - // "-debug-log", ateompath.RunscDebugLogDir(r.actorUID, containerName)+"/", + // "-debug-log", filepath.Join(r.actorDirs.GetRootDir(), "runsc-debug-logs", containerName) + "/", // "-debug-to-user-log", // "-log-packets", // "-strace", - "-root", ateompath.RunSCStateDir(r.actorUID), + "-root", runscStateDir(r.actorDirs), "fscheckpoint", "-image-path", checkpointPath, } @@ -216,7 +217,7 @@ func (r *runsc) pauseArgs(containerName string) []string { return []string{ "-log-format", "json", "--alsologtostderr", - "-root", ateompath.RunSCStateDir(r.actorUID), + "-root", runscStateDir(r.actorDirs), "pause", containerName, } @@ -241,7 +242,7 @@ func (r *runsc) resumeArgs(containerName string) []string { return []string{ "-log-format", "json", "--alsologtostderr", - "-root", ateompath.RunSCStateDir(r.actorUID), + "-root", runscStateDir(r.actorDirs), "resume", containerName, } @@ -273,19 +274,19 @@ func (r *runsc) cmdRestore(ctx context.Context, out io.Writer, containerName, ch "-log-format", "json", "--alsologtostderr", // "-debug", - // "-debug-log", ateompath.RunscDebugLogDir(r.actorUID, containerName)+"/", + // "-debug-log", filepath.Join(r.actorDirs.GetRootDir(), "runsc-debug-logs", containerName) + "/", // "-debug-to-user-log", // "-log-packets", // "-strace", - "-root", ateompath.RunSCStateDir(r.actorUID), + "-root", runscStateDir(r.actorDirs), // Match cmdCreate: size the restored sentry from the cgroup CPU quota. "--cpu-num-from-quota", } restoreArgs = append(restoreArgs, "restore", - "-bundle", ateompath.OCIBundlePath(r.actorUID, containerName), + "-bundle", ociBundlePath(r.actorDirs, containerName), "-image-path", checkpointPath, - "-pid-file", ateompath.PIDFilePath(r.actorUID, containerName), + "-pid-file", pidFilePath(r.actorDirs, containerName), "-background", "-detach", containerName, @@ -306,7 +307,7 @@ func (r *runsc) cmdDelete(ctx context.Context, containerName string) error { "-log-format", "json", "--alsologtostderr", // "-debug", - "-root", ateompath.RunSCStateDir(r.actorUID), + "-root", runscStateDir(r.actorDirs), "delete", "-force", containerName, @@ -326,7 +327,7 @@ func (r *runsc) cmdState(ctx context.Context, containerName string) error { r.path, "-log-format", "json", "--alsologtostderr", - "-root", ateompath.RunSCStateDir(r.actorUID), + "-root", runscStateDir(r.actorDirs), "state", containerName, ) @@ -345,7 +346,7 @@ func (r *runsc) cmdList(ctx context.Context) ([]string, error) { r.path, "-log-format", "json", "--alsologtostderr", - "-root", ateompath.RunSCStateDir(r.actorUID), + "-root", runscStateDir(r.actorDirs), "list", "-quiet", ) @@ -364,7 +365,7 @@ func (r *runsc) killArgs(containerName, signal string) []string { return []string{ "-log-format", "json", "--alsologtostderr", - "-root", ateompath.RunSCStateDir(r.actorUID), + "-root", runscStateDir(r.actorDirs), "kill", containerName, signal, @@ -391,7 +392,7 @@ func (r *runsc) waitArgs(containerName string) []string { return []string{ "-log-format", "json", "--alsologtostderr", - "-root", ateompath.RunSCStateDir(r.actorUID), + "-root", runscStateDir(r.actorDirs), "wait", containerName, } diff --git a/cmd/ateom-gvisor/runsc_test.go b/cmd/ateom-gvisor/runsc_test.go index c46373cd70..8edd550fd4 100644 --- a/cmd/ateom-gvisor/runsc_test.go +++ b/cmd/ateom-gvisor/runsc_test.go @@ -20,21 +20,24 @@ import ( "reflect" "testing" - "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/ocispec" + "github.com/agent-substrate/substrate/internal/proto/ateompb" ) +var testActorDirs = &ateompb.ActorDirs{RootDir: "/node/actors/test-actor-123"} + func TestKillArgs(t *testing.T) { r := &runsc{ - path: "/usr/bin/runsc", - actorUID: "test-actor-123", + path: "/usr/bin/runsc", + actorUID: "test-actor-123", + actorDirs: testActorDirs, } got := r.killArgs("my-container", "SIGTERM") want := []string{ "-log-format", "json", "--alsologtostderr", - "-root", ateompath.RunSCStateDir("test-actor-123"), + "-root", "/node/actors/test-actor-123/runsc-state", "kill", "my-container", "SIGTERM", @@ -47,15 +50,16 @@ func TestKillArgs(t *testing.T) { func TestWaitArgs(t *testing.T) { r := &runsc{ - path: "/usr/bin/runsc", - actorUID: "test-actor-123", + path: "/usr/bin/runsc", + actorUID: "test-actor-123", + actorDirs: testActorDirs, } got := r.waitArgs("my-container") want := []string{ "-log-format", "json", "--alsologtostderr", - "-root", ateompath.RunSCStateDir("test-actor-123"), + "-root", "/node/actors/test-actor-123/runsc-state", "wait", "my-container", } @@ -67,15 +71,16 @@ func TestWaitArgs(t *testing.T) { func TestPauseArgs(t *testing.T) { r := &runsc{ - path: "/usr/bin/runsc", - actorUID: "test-actor-123", + path: "/usr/bin/runsc", + actorUID: "test-actor-123", + actorDirs: testActorDirs, } got := r.pauseArgs(ocispec.PauseContainer) want := []string{ "-log-format", "json", "--alsologtostderr", - "-root", ateompath.RunSCStateDir("test-actor-123"), + "-root", "/node/actors/test-actor-123/runsc-state", "pause", ocispec.PauseContainer, } @@ -87,15 +92,16 @@ func TestPauseArgs(t *testing.T) { func TestResumeArgs(t *testing.T) { r := &runsc{ - path: "/usr/bin/runsc", - actorUID: "test-actor-123", + path: "/usr/bin/runsc", + actorUID: "test-actor-123", + actorDirs: testActorDirs, } got := r.resumeArgs(ocispec.PauseContainer) want := []string{ "-log-format", "json", "--alsologtostderr", - "-root", ateompath.RunSCStateDir("test-actor-123"), + "-root", "/node/actors/test-actor-123/runsc-state", "resume", ocispec.PauseContainer, } diff --git a/cmd/ateom-gvisor/sandboxnet.go b/cmd/ateom-gvisor/sandboxnet.go index 03263a26e3..dbd693cb30 100644 --- a/cmd/ateom-gvisor/sandboxnet.go +++ b/cmd/ateom-gvisor/sandboxnet.go @@ -27,17 +27,17 @@ import ( "github.com/agent-substrate/substrate/internal/ateomnet" "github.com/agent-substrate/substrate/internal/ateomnet/dns" - "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/internal/proto/ateompb" ) // actorResolvConf writes the resolver bind source outside the actor's rootfs. -func actorResolvConf(actorUID string) (string, error) { +func actorResolvConf(actorDirs *ateompb.ActorDirs) (string, error) { pod, err := os.ReadFile("/etc/resolv.conf") if err != nil { return "", fmt.Errorf("reading the worker pod resolv.conf: %w", err) } - path := ateompath.ActorResolvConfPath(actorUID) + path := resolvConfPath(actorDirs) if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { return "", fmt.Errorf("creating the actor directory: %w", err) } @@ -49,8 +49,11 @@ func actorResolvConf(actorUID string) (string, error) { // removeActorResolvConf drops the file with the actor's network; atelet's // per-activation reset clears directories under the actor's path, not files. -func removeActorResolvConf(ctx context.Context, actorUID string) { - if err := os.Remove(ateompath.ActorResolvConfPath(actorUID)); err != nil && !errors.Is(err, fs.ErrNotExist) { +func removeActorResolvConf(ctx context.Context, path string) { + if path == "" { + return + } + if err := os.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) { slog.WarnContext(ctx, "Failed to remove the actor resolv.conf", slog.Any("err", err)) } } diff --git a/internal/ateompath/ateompath.go b/internal/ateompath/ateompath.go index 7f3d5b79b7..e9322b77a8 100644 --- a/internal/ateompath/ateompath.go +++ b/internal/ateompath/ateompath.go @@ -30,20 +30,6 @@ func ActorPath(actorUID string) string { ) } -// ActorResolvConfPath is the resolver bind source outside the actor's rootfs. -// ateom-gvisor writes it under ActorDirs.root_dir. -func ActorResolvConfPath(actorUID string) string { - return filepath.Join(ActorPath(actorUID), "resolv.conf") -} - -// RunSCStateDir is runsc --root for the actor's sandbox, under ActorDirs.root_dir. -func RunSCStateDir(actorUID string) string { - return filepath.Join( - ActorPath(actorUID), - "runsc-state", - ) -} - // OCIBundleDir is ActorDirs.oci_bundle_dir. func OCIBundleDir(actorUID string) string { return filepath.Join( @@ -66,14 +52,6 @@ func ImageVolumeMountPath(actorUID, containerName, volumeName string) string { return filepath.Join(OCIBundlePath(actorUID, containerName), "volumes", volumeName) } -func RunscDebugLogDir(actorUID, containerName string) string { - return filepath.Join( - ActorPath(actorUID), - "runsc-debug-logs", - containerName, - ) -} - // CheckpointStateDir is ActorDirs.checkpoint_dir. func CheckpointStateDir(actorUID string) string { return filepath.Join( @@ -129,21 +107,6 @@ func RestoreStateDir(actorUID string) string { ) } -// PIDFileDir is where runsc writes .pid, under ActorDirs.root_dir. -func PIDFileDir(actorUID string) string { - return filepath.Join( - ActorPath(actorUID), - "pidfiles", - ) -} - -func PIDFilePath(actorUID, containerName string) string { - return filepath.Join( - PIDFileDir(actorUID), - containerName+".pid", - ) -} - // VolumesDir is ActorDirs.volumes_dir. func VolumesDir(actorUID string) string { return filepath.Join( diff --git a/internal/resources/validate.go b/internal/resources/validate.go index 829319e51c..b95b56f106 100644 --- a/internal/resources/validate.go +++ b/internal/resources/validate.go @@ -19,8 +19,10 @@ import ( "fmt" "net/netip" "net/url" + "path/filepath" "strings" + "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/api/validate/content" "k8s.io/apimachinery/pkg/util/validation/field" @@ -87,6 +89,37 @@ func ValidateAteomUID(targetAteomUID string) error { return nil } +// ValidateActorDirs checks that every directory atelet passes to ateom is +// set, absolute and clean. +func ValidateActorDirs(actorDirs *ateompb.ActorDirs, fldPath *field.Path) field.ErrorList { + if actorDirs == nil { + return field.ErrorList{field.Required(fldPath, "")} + } + var errs field.ErrorList + for _, actorDir := range []struct{ name, path string }{ + {"root_dir", actorDirs.GetRootDir()}, + {"oci_bundle_dir", actorDirs.GetOciBundleDir()}, + {"checkpoint_dir", actorDirs.GetCheckpointDir()}, + {"restore_dir", actorDirs.GetRestoreDir()}, + {"durable_dir_volume_mounts_dir", actorDirs.GetDurableDirVolumeMountsDir()}, + {"system_info_volume_roots_dir", actorDirs.GetSystemInfoVolumeRootsDir()}, + {"volumes_dir", actorDirs.GetVolumesDir()}, + } { + errs = append(errs, validateAbsDir(actorDir.path, fldPath.Child(actorDir.name))...) + } + return errs +} + +func validateAbsDir(dir string, fldPath *field.Path) field.ErrorList { + if dir == "" { + return field.ErrorList{field.Required(fldPath, "")} + } + if !filepath.IsAbs(dir) || filepath.Clean(dir) != dir { + return field.ErrorList{field.Invalid(fldPath, dir, "must be an absolute, clean path")} + } + return nil +} + // ValidateContainerNames ensures every application container name is safe to // use as an OCI bundle path component. Each must be a DNS-1123 label (no // separator or ".."), must not be the reserved "pause" name (which would diff --git a/internal/resources/validate_test.go b/internal/resources/validate_test.go index 7b718b454f..4512475d4c 100644 --- a/internal/resources/validate_test.go +++ b/internal/resources/validate_test.go @@ -19,6 +19,7 @@ import ( "strings" "testing" + "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -272,3 +273,53 @@ func TestValidateUUID(t *testing.T) { }) } } + +func testActorDirs() *ateompb.ActorDirs { + return &ateompb.ActorDirs{ + RootDir: "/node/actors/a", + OciBundleDir: "/node/actors/a/bundles", + CheckpointDir: "/node/actors/a/checkpoint-state", + RestoreDir: "/node/actors/a/restore-state", + DurableDirVolumeMountsDir: "/node/actors/a/durable-dir", + SystemInfoVolumeRootsDir: "/node/actors/a/system-info", + VolumesDir: "/node/actors/a/volumes", + } +} + +func TestValidateActorDirs(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*ateompb.ActorDirs) *ateompb.ActorDirs + // wantField is the field the one expected error names; empty means valid. + wantField string + }{ + {"valid", func(actorDirs *ateompb.ActorDirs) *ateompb.ActorDirs { return actorDirs }, ""}, + {"nil", func(*ateompb.ActorDirs) *ateompb.ActorDirs { return nil }, "actor_dirs"}, + {"missing dir", func(actorDirs *ateompb.ActorDirs) *ateompb.ActorDirs { actorDirs.RestoreDir = ""; return actorDirs }, "actor_dirs.restore_dir"}, + {"relative dir", func(actorDirs *ateompb.ActorDirs) *ateompb.ActorDirs { + actorDirs.OciBundleDir = "bundles" + return actorDirs + }, "actor_dirs.oci_bundle_dir"}, + {"unclean dir", func(actorDirs *ateompb.ActorDirs) *ateompb.ActorDirs { + actorDirs.CheckpointDir = "/node/actors/a/../b/checkpoint-state" + return actorDirs + }, "actor_dirs.checkpoint_dir"}, + {"relative root", func(actorDirs *ateompb.ActorDirs) *ateompb.ActorDirs { + actorDirs.RootDir = "node/actors/a" + return actorDirs + }, "actor_dirs.root_dir"}, + } { + t.Run(tc.name, func(t *testing.T) { + errs := ValidateActorDirs(tc.mutate(testActorDirs()), field.NewPath("actor_dirs")) + if tc.wantField == "" { + if len(errs) != 0 { + t.Fatalf("ValidateActorDirs() = %v, want no errors", errs) + } + return + } + if len(errs) != 1 || errs[0].Field != tc.wantField { + t.Fatalf("ValidateActorDirs() = %v, want one error on %s", errs, tc.wantField) + } + }) + } +}