diff --git a/TESTING.md b/TESTING.md index 58bb638831..dc3f7a2013 100644 --- a/TESTING.md +++ b/TESTING.md @@ -849,7 +849,7 @@ cargo test --manifest-path e2e/rust/Cargo.toml --features e2e --test sync Run a single Docker-only test directly with cargo: ```shell -cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker --test docker_preflight +cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker --test docker_corporate_proxy ``` The harness (`e2e/rust/src/harness/`) provides: diff --git a/crates/openshell-cli/tests/doctor_integration.rs b/crates/openshell-cli/tests/doctor_integration.rs new file mode 100644 index 0000000000..343dc7a9c2 --- /dev/null +++ b/crates/openshell-cli/tests/doctor_integration.rs @@ -0,0 +1,142 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `openshell doctor check` against a fake `docker` on `PATH`, so the result +//! doesn't depend on the host's Docker. + +#![cfg(unix)] + +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +struct Doctor { + combined: String, + code: i32, + elapsed: Duration, +} + +/// Run `openshell doctor check` with only `bin` on `PATH`. +fn doctor_check(bin: &Path) -> Doctor { + let home = tempfile::tempdir().expect("create isolated config dir"); + let start = Instant::now(); + let output = Command::new(env!("CARGO_BIN_EXE_openshell")) + .args(["doctor", "check"]) + .env("XDG_CONFIG_HOME", home.path()) + .env("HOME", home.path()) + .env("PATH", bin) + .env("NO_COLOR", "1") + .env_remove("DOCKER_HOST") + .env_remove("OPENSHELL_GATEWAY") + .env_remove("OPENSHELL_GATEWAY_ENDPOINT") + .stdin(Stdio::null()) + .output() + .expect("run openshell doctor check"); + Doctor { + combined: format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ), + code: output.status.code().unwrap_or(-1), + elapsed: start.elapsed(), + } +} + +/// Write an executable `docker` script to `bin` that records its arguments +/// in `bin/docker-args` and then runs `body`. +fn fake_docker(bin: &Path, body: &str) { + let docker = bin.join("docker"); + let args = bin.join("docker-args"); + fs::write( + &docker, + format!( + "#!/bin/sh\nprintf '%s\\n' \"$*\" > '{}'\n{body}\n", + args.display() + ), + ) + .expect("write fake docker"); + fs::set_permissions(&docker, fs::Permissions::from_mode(0o755)).expect("chmod fake docker"); +} + +fn docker_args(bin: &Path) -> String { + fs::read_to_string(bin.join("docker-args")).unwrap_or_default() +} + +#[test] +fn doctor_check_passes_when_docker_info_succeeds() { + let bin = tempfile::tempdir().expect("create fake bin dir"); + fake_docker(bin.path(), "echo 27.3.1"); + + let doctor = doctor_check(bin.path()); + assert_eq!(doctor.code, 0, "doctor check failed:\n{}", doctor.combined); + assert_eq!( + docker_args(bin.path()).trim(), + "info --format {{.ServerVersion}}", + "doctor check ran docker with unexpected arguments" + ); + for expected in ["Docker", "ok (version 27.3.1)", "All checks passed"] { + assert!( + doctor.combined.contains(expected), + "expected {expected:?} in doctor check output:\n{}", + doctor.combined + ); + } +} + +#[test] +fn doctor_check_fails_fast_when_docker_info_fails() { + let bin = tempfile::tempdir().expect("create fake bin dir"); + fake_docker( + bin.path(), + "echo 'Cannot connect to the Docker daemon' >&2\nexit 1", + ); + + let doctor = doctor_check(bin.path()); + assert_ne!( + doctor.code, 0, + "doctor check passed although docker info failed:\n{}", + doctor.combined + ); + assert!( + doctor.elapsed < Duration::from_secs(10), + "doctor check took {:?}", + doctor.elapsed + ); + for expected in [ + "Docker", + "FAILED", + "docker info failed", + "Cannot connect to the Docker daemon", + ] { + assert!( + doctor.combined.contains(expected), + "expected {expected:?} in doctor check output:\n{}", + doctor.combined + ); + } + assert!( + !doctor.combined.contains("All checks passed"), + "doctor check reported success:\n{}", + doctor.combined + ); +} + +#[test] +fn doctor_check_fails_without_docker_on_path() { + let bin = tempfile::tempdir().expect("create empty bin dir"); + + let doctor = doctor_check(bin.path()); + assert_ne!( + doctor.code, 0, + "doctor check passed without docker:\n{}", + doctor.combined + ); + assert!( + doctor.combined.contains("failed to execute docker info"), + "doctor check did not report that docker could not run:\n{}", + doctor.combined + ); +} diff --git a/crates/openshell-gateway/tests/proxy_config_preflight.rs b/crates/openshell-gateway/tests/proxy_config_preflight.rs new file mode 100644 index 0000000000..d3d1917399 --- /dev/null +++ b/crates/openshell-gateway/tests/proxy_config_preflight.rs @@ -0,0 +1,136 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! An incoherent driver proxy table is fatal, rather than leaving every dial +//! direct while looking configured. A bypass list without a proxy URL means +//! the operator believed proxying was in effect. + +// Only Docker, Podman and the VM driver have a proxy table. +#![cfg(all( + unix, + any( + feature = "compute-driver-docker", + feature = "compute-driver-podman", + feature = "compute-driver-vm" + ) +))] + +use std::fs; +use std::path::PathBuf; +use std::process::Output; +use std::time::Duration; + +/// Drivers with an upstream proxy table, compiled into this gateway. +const DRIVERS: &[&str] = &[ + #[cfg(feature = "compute-driver-docker")] + "docker", + #[cfg(feature = "compute-driver-podman")] + "podman", + #[cfg(feature = "compute-driver-vm")] + "vm", +]; + +struct Fixture { + root: tempfile::TempDir, + config: PathBuf, +} + +impl Fixture { + /// A gateway config that selects `driver` and sets only `no_proxy` in its + /// table. + fn incoherent(driver: &str) -> Self { + let root = tempfile::tempdir().expect("fixture root"); + let config = root.path().join("gateway.toml"); + fs::write( + &config, + format!( + "[openshell] +version = 2 +[openshell.gateway] +disable_tls = true +bind_address = \"127.0.0.1:0\" +compute_driver = {driver:?} +[openshell.gateway.auth] +allow_unauthenticated_users = true +[openshell.drivers.{driver}] +no_proxy = \"10.0.0.0/8\" +" + ), + ) + .expect("gateway configuration"); + Self { root, config } + } + + async fn run(&self, args: &[&str]) -> Output { + let mut command = tokio::process::Command::new(env!("CARGO_BIN_EXE_openshell-gateway")); + command + .env_clear() + .env("HOME", self.root.path()) + .env("PATH", "/usr/bin:/bin") + .env("NO_COLOR", "1") + .env("XDG_CONFIG_HOME", self.root.path().join("config-home")) + .env("XDG_STATE_HOME", self.root.path().join("state-home")) + .args(args) + .kill_on_drop(true); + tokio::time::timeout(Duration::from_secs(30), command.output()) + .await + .expect("the gateway must exit instead of starting") + .expect("run openshell-gateway") + } + + fn config_path(&self) -> &str { + self.config.to_str().expect("UTF-8 config path") + } +} + +fn combined(output: &Output) -> String { + format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) +} + +#[tokio::test] +async fn preflight_rejects_no_proxy_without_proxy() { + for driver in DRIVERS { + let fixture = Fixture::incoherent(driver); + let original = fs::read(&fixture.config).expect("original config"); + let output = fixture + .run(&["config", "preflight", "--path", fixture.config_path()]) + .await; + let report = combined(&output); + assert!( + !output.status.success(), + "preflight accepted an incoherent [openshell.drivers.{driver}] proxy table:\n{report}" + ); + assert!( + report.contains("category=malformed"), + "preflight did not classify the {driver} config as malformed:\n{report}" + ); + assert_eq!( + fs::read(&fixture.config).expect("config after preflight"), + original, + "preflight changed the {driver} config" + ); + } +} + +#[tokio::test] +async fn startup_rejects_no_proxy_without_proxy_and_names_the_keys() { + for driver in DRIVERS { + let fixture = Fixture::incoherent(driver); + let output = fixture.run(&["--config", fixture.config_path()]).await; + let report = combined(&output); + assert!( + !output.status.success(), + "the gateway started with an incoherent [openshell.drivers.{driver}] proxy table:\n{report}" + ); + // Matched on the key names, not the wording, so error wrapping can + // change. + assert!( + report.contains("no_proxy") && report.contains("https_proxy"), + "the {driver} startup error does not name the offending keys:\n{report}" + ); + } +} diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 19ce583732..e73648fd36 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -89,11 +89,6 @@ name = "vm_bind_mount" path = "tests/vm_bind_mount.rs" required-features = ["e2e-vm"] -[[test]] -name = "docker_preflight" -path = "tests/docker_preflight.rs" -required-features = ["e2e-docker"] - [[test]] name = "docker_corporate_proxy" path = "tests/docker_corporate_proxy.rs" @@ -144,11 +139,6 @@ name = "podman_corporate_proxy" path = "tests/podman_corporate_proxy.rs" required-features = ["e2e-podman"] -[[test]] -name = "podman_oci_identity" -path = "tests/podman_oci_identity.rs" -required-features = ["e2e-podman"] - [[test]] name = "podman_resource_limits" path = "tests/podman_resource_limits.rs" @@ -234,11 +224,6 @@ name = "workspace_lifecycle" path = "tests/workspace_lifecycle.rs" required-features = ["e2e"] -[[test]] -name = "sandbox_templates" -path = "tests/sandbox_templates.rs" -required-features = ["e2e"] - [[test]] name = "supervisor_middleware_http_hooks" path = "tests/supervisor_middleware_http_hooks.rs" @@ -316,7 +301,3 @@ module_name_repetitions = "allow" must_use_candidate = "allow" missing_errors_doc = "allow" missing_panics_doc = "allow" -[[test]] -name = "live_internet_traffic_perf" -path = "tests/live_internet_traffic_perf.rs" -required-features = ["e2e-host-gateway"] diff --git a/e2e/rust/e2e-podman.sh b/e2e/rust/e2e-podman.sh index e718f0afea..d5a54741b7 100755 --- a/e2e/rust/e2e-podman.sh +++ b/e2e/rust/e2e-podman.sh @@ -24,15 +24,12 @@ source "${ROOT}/e2e/support/conformance.sh" PODMAN_CI_TESTS=( bypass_detection ephemeral_cleanup - core_dump_hardening - default_image driver_config_volume edge_tunnel_e2e forward_proxy_graphql_l7 forward_proxy_jsonrpc_l7 forward_proxy_l7_bypass host_gateway_alias - landlock live_policy_update local_driver_token_restart no_proxy @@ -41,9 +38,7 @@ PODMAN_CI_TESTS=( podman_corporate_proxy podman_gateway_start podman_host_gateway - podman_oci_identity provider_token_exchange - upload_backpressure ) if [ -n "${E2E_TEST}" ] && [ -n "${E2E_TEST_SET}" ]; then diff --git a/e2e/rust/src/harness/sandbox.rs b/e2e/rust/src/harness/sandbox.rs index c0f0c1aedc..9f57a29daa 100644 --- a/e2e/rust/src/harness/sandbox.rs +++ b/e2e/rust/src/harness/sandbox.rs @@ -129,18 +129,6 @@ impl SandboxGuard { /// Returns an error if the CLI exits with a non-zero status or the sandbox /// name cannot be parsed from the output. pub async fn create(args: &[&str]) -> Result { - Self::create_inner(args, true).await - } - - /// Create a sandbox using the gateway's configured default image. - /// - /// Most E2E tests use [`Self::create`], which supplies the tool-capable E2E - /// image. This variant is reserved for coverage of the product default. - pub async fn create_with_gateway_default(args: &[&str]) -> Result { - Self::create_inner(args, false).await - } - - async fn create_inner(args: &[&str], use_test_image: bool) -> Result { let separator = args.iter().position(|arg| *arg == "--"); let (create_args, command) = separator.map_or((args, &[][..]), |index| { (&args[..index], &args[index + 1..]) @@ -156,9 +144,7 @@ impl SandboxGuard { let mut cmd = openshell_cmd(); cmd.arg("sandbox").arg("create").arg("--detach"); add_unique_name_if_missing(&mut cmd, create_args); - if use_test_image { - add_test_image_if_missing(&mut cmd, create_args); - } + add_test_image_if_missing(&mut cmd, create_args); for arg in create_args { cmd.arg(arg); } diff --git a/e2e/rust/tests/core_dump_hardening.rs b/e2e/rust/tests/core_dump_hardening.rs deleted file mode 100644 index a400b6c218..0000000000 --- a/e2e/rust/tests/core_dump_hardening.rs +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -use openshell_e2e::harness::sandbox::SandboxGuard; - -#[tokio::test] -async fn sandbox_processes_disable_core_dumps() { - let mut sb = SandboxGuard::create(&[ - "--", - "sh", - "-lc", - "test \"$(ulimit -c)\" = 0 && echo core-limit-ok", - ]) - .await - .expect("sandbox create should succeed"); - - assert!( - sb.create_output.contains("core-limit-ok"), - "expected sandbox output to confirm core dumps are disabled:\n{}", - sb.create_output, - ); - - sb.cleanup().await; -} diff --git a/e2e/rust/tests/default_image.rs b/e2e/rust/tests/default_image.rs deleted file mode 100644 index 6a1dbb3cd8..0000000000 --- a/e2e/rust/tests/default_image.rs +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E coverage for the default NVIDIA Ubuntu workload image. - -use openshell_e2e::harness::output::strip_ansi; -use openshell_e2e::harness::sandbox::SandboxGuard; - -#[tokio::test] -async fn sandbox_from_default_image() { - let mut guard = SandboxGuard::create_with_gateway_default(&["--", "cat", "/etc/os-release"]) - .await - .expect("sandbox create from default image"); - - let clean_output = strip_ansi(&guard.create_output); - assert!( - clean_output.contains("ID=ubuntu"), - "expected Ubuntu OS release info in sandbox output:\n{clean_output}" - ); - - guard.cleanup().await; -} - -#[tokio::test] -async fn sandbox_from_explicit_nvidia_ubuntu_image() { - let image = "nvcr.io/nvidia/base/ubuntu:24.04"; - let mut guard = SandboxGuard::create(&["--from", image, "--", "cat", "/etc/os-release"]) - .await - .expect("sandbox create from explicit NVIDIA Ubuntu image"); - - let clean_output = strip_ansi(&guard.create_output); - assert!( - clean_output.contains("ID=ubuntu"), - "expected Ubuntu OS release info in sandbox output:\n{clean_output}" - ); - - guard.cleanup().await; -} diff --git a/e2e/rust/tests/docker_corporate_proxy.rs b/e2e/rust/tests/docker_corporate_proxy.rs index 85584b966f..4cd238a0ff 100644 --- a/e2e/rust/tests/docker_corporate_proxy.rs +++ b/e2e/rust/tests/docker_corporate_proxy.rs @@ -23,8 +23,6 @@ //! 5. An `https://` proxy works when its CA is supplied via `proxy_ca_bundle`. //! 6. A TLS-intercepting proxy CA is trusted by both inspected supervisor //! upstream TLS and a raw workload TLS connection. -//! 7. An incoherent setting is fatal at gateway startup rather than -//! degrading to a direct dial. //! //! Fixtures run as host processes and are reached by the host-networked //! supervisor. `host.openshell.internal` is normalized to the gateway host @@ -1142,39 +1140,3 @@ async fn docker_corporate_proxy_trusts_intercepted_destination_tls() { .await .expect("restore gateway config"); } - -#[tokio::test] -#[serial(docker_corporate_proxy)] -async fn docker_corporate_proxy_rejects_incoherent_configuration() { - if !should_run("corporate proxy fail-closed test") { - return; - } - - // A bypass list without a proxy URL means the operator believed proxying - // was in effect. Accepting it would leave every dial direct while looking - // configured, so the gateway must refuse to start instead. - let mut gateway_config = GatewayProxyConfig::apply_raw("no_proxy = \"10.0.0.0/8\"\n") - .expect("apply incoherent corporate proxy gateway config"); - - let health = wait_for_healthy(Duration::from_secs(20)).await; - assert!( - health.is_err(), - "gateway must not serve traffic with an incoherent [openshell.drivers.docker] proxy table" - ); - - let log_path = std::env::var("OPENSHELL_E2E_GATEWAY_LOG").expect("gateway log path"); - let log = std::fs::read_to_string(&log_path).expect("read gateway log"); - // The message must name the offending key rather than surfacing as an - // opaque driver-readiness timeout. Matched loosely on the key names so - // this does not break on error-wrapping changes. - assert!( - log.contains("no_proxy") && log.contains("https_proxy"), - "the startup error must name the offending key; gateway log tail:\n{}", - log.lines().rev().take(40).collect::>().join("\n") - ); - - gateway_config - .restore() - .await - .expect("restore gateway config"); -} diff --git a/e2e/rust/tests/docker_preflight.rs b/e2e/rust/tests/docker_preflight.rs deleted file mode 100644 index bf05a132d9..0000000000 --- a/e2e/rust/tests/docker_preflight.rs +++ /dev/null @@ -1,171 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Doctor Docker preflight e2e tests. -//! -//! These tests verify that `openshell doctor check` reports actionable guidance -//! when Docker is not available. -//! -//! The tests do NOT require a running gateway or Docker — they intentionally -//! point `DOCKER_HOST` at a non-existent socket to simulate Docker being -//! unavailable. - -use std::process::Stdio; -use std::time::Instant; -use std::{env, fs}; - -#[cfg(unix)] -use std::os::unix::fs::PermissionsExt; - -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::output::strip_ansi; - -/// Run `openshell ` in an isolated environment where Docker is -/// guaranteed to be unreachable. -/// -/// Sets `DOCKER_HOST` to a non-existent socket so the preflight check -/// fails immediately regardless of the host's Docker configuration. -async fn run_without_docker(args: &[&str]) -> (String, i32, std::time::Duration) { - let tmpdir = tempfile::tempdir().expect("create isolated config dir"); - let bin_dir = tmpdir.path().join("bin"); - fs::create_dir(&bin_dir).expect("create fake bin dir"); - let fake_docker = bin_dir.join("docker"); - fs::write( - &fake_docker, - "#!/bin/sh\n\ - echo 'Cannot connect to Docker daemon. Check DOCKER_HOST and run docker info.' >&2\n\ - exit 1\n", - ) - .expect("write fake docker"); - #[cfg(unix)] - fs::set_permissions(&fake_docker, fs::Permissions::from_mode(0o755)) - .expect("chmod fake docker"); - - let old_path = env::var("PATH").unwrap_or_default(); - let path = format!("{}:{old_path}", bin_dir.display()); - let start = Instant::now(); - - let mut cmd = openshell_cmd(); - cmd.args(args) - .env("XDG_CONFIG_HOME", tmpdir.path()) - .env("HOME", tmpdir.path()) - .env("PATH", path) - .env("DOCKER_HOST", "unix:///tmp/openshell-e2e-nonexistent.sock") - .env_remove("OPENSHELL_GATEWAY") - .env_remove("OPENSHELL_GATEWAY_ENDPOINT") - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell"); - let elapsed = start.elapsed(); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - let code = output.status.code().unwrap_or(-1); - (combined, code, elapsed) -} - -// ------------------------------------------------------------------- -// doctor check: validates system prerequisites -// ------------------------------------------------------------------- - -/// `openshell doctor check` with Docker unavailable should fail fast -/// and report the Docker check as FAILED. -#[tokio::test] -async fn doctor_check_fails_without_docker() { - let (output, code, elapsed) = run_without_docker(&["doctor", "check"]).await; - - assert_ne!( - code, 0, - "doctor check should fail when Docker is unavailable, output:\n{output}" - ); - - assert!( - elapsed.as_secs() < 10, - "doctor check should complete quickly (took {}s)", - elapsed.as_secs() - ); - - let clean = strip_ansi(&output); - assert!( - clean.contains("FAILED"), - "doctor check should report Docker as FAILED:\n{clean}" - ); -} - -/// `openshell doctor check` output should include the check label -/// so the user knows what was tested. -#[tokio::test] -async fn doctor_check_output_shows_docker_label() { - let (output, _, _) = run_without_docker(&["doctor", "check"]).await; - let clean = strip_ansi(&output); - - assert!( - clean.contains("Docker"), - "doctor check output should include 'Docker' label:\n{clean}" - ); -} - -/// `openshell doctor check` with Docker unavailable should include -/// actionable guidance in the error output. -#[tokio::test] -async fn doctor_check_error_includes_guidance() { - let (output, code, _) = run_without_docker(&["doctor", "check"]).await; - - assert_ne!(code, 0); - let clean = strip_ansi(&output); - - assert!( - clean.contains("DOCKER_HOST"), - "doctor check error should mention DOCKER_HOST:\n{clean}" - ); - assert!( - clean.contains("docker info"), - "doctor check error should suggest 'docker info':\n{clean}" - ); -} - -/// When Docker IS available, `openshell doctor check` should pass and -/// report the version. -/// -/// This test only runs when Docker is actually reachable on the host -/// (i.e., it will pass in CI with Docker but be skipped locally if -/// Docker is not running). We detect this by checking if the default -/// socket exists. -#[tokio::test] -async fn doctor_check_passes_with_docker() { - if !std::path::Path::new("/var/run/docker.sock").exists() { - eprintln!("skipping: /var/run/docker.sock not found"); - return; - } - - let tmpdir = tempfile::tempdir().expect("create isolated config dir"); - let mut cmd = openshell_cmd(); - cmd.args(["doctor", "check"]) - .env("XDG_CONFIG_HOME", tmpdir.path()) - .env("HOME", tmpdir.path()) - .env_remove("OPENSHELL_GATEWAY") - .env_remove("OPENSHELL_GATEWAY_ENDPOINT") - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - let code = output.status.code().unwrap_or(-1); - let clean = strip_ansi(&combined); - - assert_eq!( - code, 0, - "doctor check should pass when Docker is available, output:\n{clean}" - ); - assert!( - clean.contains("All checks passed"), - "doctor check should report success:\n{clean}" - ); - assert!( - clean.contains("ok"), - "doctor check should show 'ok' for Docker:\n{clean}" - ); -} diff --git a/e2e/rust/tests/internet_network_perf.rs b/e2e/rust/tests/internet_network_perf.rs deleted file mode 100644 index 70c0f1af25..0000000000 --- a/e2e/rust/tests/internet_network_perf.rs +++ /dev/null @@ -1,170 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Opt-in Internet benchmark for the complete sandbox-to-supervisor data path. - -#![cfg(feature = "e2e-host-gateway")] - -use std::io::Write as _; -use std::process::Stdio; - -use openshell_e2e::harness::sandbox::SandboxGuard; -use tempfile::NamedTempFile; - -const BENCHMARK: &str = r#" -import http.client -import json -import os -import socket -import ssl -import statistics -import time - -HOST = "example.com" -ITERATIONS = int(os.environ.get("OPENSHELL_INET_PERF_ITERATIONS", "20")) - -def percentile(values, fraction): - values = sorted(values) - return values[min(len(values) - 1, int((len(values) - 1) * fraction))] - -def measure(name, operation, iterations=ITERATIONS): - operation() - samples = [] - started = time.perf_counter_ns() - for _ in range(iterations): - before = time.perf_counter_ns() - operation() - samples.append((time.perf_counter_ns() - before) / 1_000_000) - elapsed = (time.perf_counter_ns() - started) / 1_000_000_000 - return { - "name": name, - "iterations": iterations, - "mean_ms": statistics.fmean(samples), - "p50_ms": percentile(samples, 0.50), - "p95_ms": percentile(samples, 0.95), - "ops_per_second": iterations / elapsed, - } - -def dns_lookup(): - result = socket.getaddrinfo(HOST, 443, socket.AF_INET, socket.SOCK_STREAM) - if not result: - raise RuntimeError("DNS returned no IPv4 addresses") - -def tcp_connect(): - with socket.create_connection((HOST, 443), timeout=10): - pass - -tls_context = ssl.create_default_context() - -def https_cold(): - connection = http.client.HTTPSConnection(HOST, 443, timeout=10, context=tls_context) - try: - connection.request("HEAD", "/", headers={"Connection": "close"}) - response = connection.getresponse() - response.read() - if response.status != 200: - raise RuntimeError(f"unexpected HTTP status {response.status}") - finally: - connection.close() - -warm_connection = http.client.HTTPSConnection(HOST, 443, timeout=10, context=tls_context) - -def https_reuse(): - warm_connection.request("HEAD", "/", headers={"Connection": "keep-alive"}) - response = warm_connection.getresponse() - response.read() - if response.status != 200: - raise RuntimeError(f"unexpected HTTP status {response.status}") - -try: - metrics = [ - measure("dns_lookup", dns_lookup), - measure("tcp_connect", tcp_connect), - measure("https_cold", https_cold, max(5, ITERATIONS // 2)), - measure("https_reuse", https_reuse), - ] -finally: - warm_connection.close() - -print(json.dumps({"metrics": metrics}, separators=(",", ":"))) -"#; - -fn write_policy() -> Result { - let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; - write!( - file, - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] - -landlock: - compatibility: best_effort - -network_policies: - internet_performance: - name: internet_performance - endpoints: - - host: example.com - port: 80 - protocol: tcp - - host: example.com - port: 443 - protocol: tcp - binaries: - - path: "/**" -"# - ) - .map_err(|error| format!("write policy: {error}"))?; - file.flush() - .map_err(|error| format!("flush policy: {error}"))?; - Ok(file) -} - -async fn run_host_benchmark() -> Result { - let output = tokio::process::Command::new("python3") - .args(["-c", BENCHMARK]) - .env("OPENSHELL_INET_PERF_ITERATIONS", "20") - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .output() - .await - .map_err(|error| format!("run host benchmark: {error}"))?; - let combined = format!( - "{}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - if !output.status.success() { - return Err(format!("host benchmark failed: {combined}")); - } - Ok(combined) -} - -#[tokio::test] -#[ignore = "manual Internet performance benchmark"] -async fn benchmark_complete_internet_path() { - let policy = write_policy().expect("write Internet benchmark policy"); - let policy_path = policy.path().to_str().expect("UTF-8 policy path"); - let sandbox = SandboxGuard::create(&["--policy", policy_path]) - .await - .expect("create benchmark sandbox"); - - for round in 1..=3 { - let host = run_host_benchmark().await.expect("host benchmark"); - println!("INTERNET_PERF host round={round} {}", host.trim()); - let mediated = sandbox - .exec(&[ - "sh", - "-c", - "OPENSHELL_INET_PERF_ITERATIONS=20 python3 -c \"$1\"", - "openshell-internet-perf", - BENCHMARK, - ]) - .await - .expect("sandbox benchmark"); - println!("INTERNET_PERF sandbox round={round} {}", mediated.trim()); - } -} diff --git a/e2e/rust/tests/landlock.rs b/e2e/rust/tests/landlock.rs deleted file mode 100644 index cfbdcd5846..0000000000 --- a/e2e/rust/tests/landlock.rs +++ /dev/null @@ -1,150 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! End-to-end coverage for Landlock filesystem enforcement. - -use std::io::Write; -use std::time::Duration; - -use openshell_e2e::harness::sandbox::SandboxGuard; -use tempfile::NamedTempFile; - -const SUCCESS_MARKER: &str = "landlock-hard-requirement-ok"; - -fn write_hard_requirement_policy() -> Result { - write_policy("hard_requirement", "/sandbox") -} - -fn write_policy(compatibility: &str, extra_path: &str) -> Result { - let mut file = - NamedTempFile::new().map_err(|error| format!("create temporary policy: {error}"))?; - let policy = r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: - - /usr - - /lib - - /etc - - /proc - read_write: - - /sandbox - - /tmp - -landlock: - compatibility: hard_requirement - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - landlock_regression: - name: landlock_regression - endpoints: - - host: example.com - port: 443 - binaries: - - path: "/**" -"#; - - let policy = policy - .replace( - "compatibility: hard_requirement", - &format!("compatibility: {compatibility}"), - ) - .replace( - " - /sandbox\n", - &format!(" - /sandbox\n - {extra_path}\n"), - ); - file.write_all(policy.as_bytes()) - .map_err(|error| format!("write temporary policy: {error}"))?; - file.flush() - .map_err(|error| format!("flush temporary policy: {error}"))?; - Ok(file) -} - -#[tokio::test] -async fn hard_requirement_accepts_enriched_device_path() { - let policy = write_hard_requirement_policy().expect("write hard-requirement policy"); - let policy_path = policy.path().to_string_lossy().into_owned(); - let script = concat!( - "set -eu; ", - "bytes=$(head -c 16 /dev/urandom | wc -c); ", - "test \"$bytes\" -eq 16; ", - "printf landlock-ok > /tmp/landlock-check; ", - "test \"$(cat /tmp/landlock-check)\" = landlock-ok; ", - "echo landlock-hard-requirement-ok", - ); - - let mut sandbox = SandboxGuard::create(&["--policy", &policy_path, "--", "sh", "-lc", script]) - .await - .expect("hard_requirement should accept the enriched /dev/urandom path"); - - assert!(sandbox.create_output.contains(SUCCESS_MARKER)); - sandbox.cleanup().await; -} - -async fn check_policy_path_replacement(compatibility: &str, replace_ancestor: bool) { - let policy = write_policy(compatibility, "/sandbox/policy-parent/leaf") - .expect("write filesystem policy"); - let policy_path = policy.path().to_string_lossy().into_owned(); - let mut sandbox = SandboxGuard::create(&["--policy", &policy_path]) - .await - .expect("create sandbox with filesystem policy"); - - sandbox - .exec(&["sh", "-c", "mkdir -p /sandbox/policy-parent/leaf"]) - .await - .expect("create declared directory under writable workspace"); - sandbox - .exec(&[ - "sh", - "-c", - "echo allowed > /sandbox/policy-parent/leaf/check", - ]) - .await - .expect("normal directory should remain usable on the next exec"); - - let script = if replace_ancestor { - "mv /sandbox/policy-parent /sandbox/original-parent; ln -s /var/tmp /sandbox/policy-parent" - } else { - "mv /sandbox/policy-parent/leaf /sandbox/original-leaf; ln -s /var/tmp /sandbox/policy-parent/leaf" - }; - sandbox - .exec(&["sh", "-c", script]) - .await - .expect("replace workspace directory"); - let result = tokio::time::timeout(Duration::from_secs(20), sandbox.exec(&["true"])).await; - // Delete through the control plane even when further workload launches fail. - sandbox.cleanup().await; - let error = result - .expect("rejected launch must close its SSH channel promptly") - .expect_err("mutable policy symlink must prevent the next launch"); - assert!( - error.to_lowercase().contains("symlink"), - "unexpected launch error: {error}" - ); -} - -#[tokio::test] -async fn best_effort_rejects_replaced_policy_path() { - check_policy_path_replacement("best_effort", false).await; -} - -#[tokio::test] -async fn hard_requirement_rejects_replaced_policy_path() { - check_policy_path_replacement("hard_requirement", false).await; -} - -#[tokio::test] -async fn best_effort_rejects_replaced_policy_ancestor() { - check_policy_path_replacement("best_effort", true).await; -} - -#[tokio::test] -async fn hard_requirement_rejects_replaced_policy_ancestor() { - check_policy_path_replacement("hard_requirement", true).await; -} diff --git a/e2e/rust/tests/live_internet_traffic_perf.rs b/e2e/rust/tests/live_internet_traffic_perf.rs deleted file mode 100644 index e8727bbce3..0000000000 --- a/e2e/rust/tests/live_internet_traffic_perf.rs +++ /dev/null @@ -1,337 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Opt-in live-Internet benchmark shaped like a short coding-agent session. - -#![cfg(feature = "e2e-host-gateway")] - -use std::io::Write as _; -use std::process::Stdio; -use std::time::Instant; - -use openshell_e2e::harness::sandbox::SandboxGuard; -use tempfile::NamedTempFile; - -const DEFAULT_ROUNDS: usize = 10; - -const BENCHMARK: &str = r#" -import concurrent.futures -import http.client -import json -import os -import shutil -import socket -import ssl -import statistics -import subprocess -import tempfile -import time -import urllib.parse - -MODE = os.environ.get("OPENSHELL_LIVE_PERF_MODE", "direct") -TLS = ssl.create_default_context() -USER_AGENT = "OpenShell-live-Internet-benchmark/1" - -METADATA_URLS = [ - "https://raw.githubusercontent.com/NVIDIA/OpenShell/main/README.md", - "https://pypi.org/pypi/requests/json", - "https://registry.npmjs.org/typescript/latest", - "https://docs.python.org/3/", -] -DNS_HOSTS = [ - "github.com", - "raw.githubusercontent.com", - "pypi.org", - "registry.npmjs.org", - "crates.io", - "docs.python.org", - "speed.cloudflare.com", -] - -def percentile(values, fraction): - values = sorted(values) - return values[min(len(values) - 1, int((len(values) - 1) * fraction))] - -def request(url, method="GET", max_bytes=None): - parsed = urllib.parse.urlsplit(url) - connection = http.client.HTTPSConnection(parsed.hostname, parsed.port or 443, timeout=20, context=TLS) - path = parsed.path or "/" - if parsed.query: - path += "?" + parsed.query - started = time.perf_counter_ns() - connection.request(method, path, headers={"User-Agent": USER_AGENT, "Connection": "close"}) - response = connection.getresponse() - first = response.read(1) - first_byte_ms = (time.perf_counter_ns() - started) / 1_000_000 - body_bytes = len(first) - while max_bytes is None or body_bytes < max_bytes: - remaining = None if max_bytes is None else max_bytes - body_bytes - chunk = response.read(65536 if remaining is None else min(65536, remaining)) - if not chunk: - break - body_bytes += len(chunk) - total_ms = (time.perf_counter_ns() - started) / 1_000_000 - status = response.status - connection.close() - if status < 200 or status >= 400: - raise RuntimeError(f"{url} returned HTTP {status}") - return {"status": status, "bytes": body_bytes, "first_byte_ms": first_byte_ms, "total_ms": total_ms} - -def metric(name, operation): - started = time.perf_counter_ns() - try: - detail = operation() - return { - "name": name, - "ok": True, - "elapsed_ms": (time.perf_counter_ns() - started) / 1_000_000, - "detail": detail, - } - except Exception as error: - return { - "name": name, - "ok": False, - "elapsed_ms": (time.perf_counter_ns() - started) / 1_000_000, - "error": f"{type(error).__name__}: {error}", - } - -def dns_set(): - samples = [] - for host in DNS_HOSTS: - started = time.perf_counter_ns() - addresses = socket.getaddrinfo(host, 443, socket.AF_UNSPEC, socket.SOCK_STREAM) - samples.append({ - "host": host, - "elapsed_ms": (time.perf_counter_ns() - started) / 1_000_000, - "addresses": len(addresses), - }) - timings = [sample["elapsed_ms"] for sample in samples] - return { - "lookups": samples, - "p50_ms": percentile(timings, 0.50), - "p95_ms": percentile(timings, 0.95), - } - -def metadata_serial(): - return {"requests": [request(url, max_bytes=2_000_000) for url in METADATA_URLS]} - -def metadata_concurrent(): - urls = METADATA_URLS + METADATA_URLS - with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool: - responses = list(pool.map(lambda url: request(url, max_bytes=2_000_000), urls)) - return {"requests": responses} - -def https_reuse(): - connection = http.client.HTTPSConnection("example.com", 443, timeout=20, context=TLS) - samples = [] - try: - for _ in range(20): - started = time.perf_counter_ns() - connection.request("HEAD", "/", headers={"User-Agent": USER_AGENT, "Connection": "keep-alive"}) - response = connection.getresponse() - response.read() - if response.status != 200: - raise RuntimeError(f"example.com returned HTTP {response.status}") - samples.append((time.perf_counter_ns() - started) / 1_000_000) - finally: - connection.close() - return { - "requests": len(samples), - "p50_ms": percentile(samples, 0.50), - "p95_ms": percentile(samples, 0.95), - "mean_ms": statistics.fmean(samples), - } - -def git_clone(): - if shutil.which("git") is None: - return {"skipped": "git is not installed in the workload image"} - with tempfile.TemporaryDirectory(prefix="openshell-live-git-") as directory: - checkout = os.path.join(directory, "sampleproject") - command = [ - "git", "-c", "advice.detachedHead=false", "clone", "--quiet", - "--depth", "1", "--filter=blob:none", - "https://github.com/pypa/sampleproject.git", checkout, - ] - output = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60, check=False) - if output.returncode != 0: - raise RuntimeError(output.stderr.decode(errors="replace")[-500:]) - files = sum(len(names) for _, _, names in os.walk(checkout)) - size = sum(os.path.getsize(os.path.join(root, name)) for root, _, names in os.walk(checkout) for name in names) - return {"files": files, "checkout_bytes": size} - -def package_download(): - metadata = request("https://pypi.org/pypi/idna/json", max_bytes=2_000_000) - connection = http.client.HTTPSConnection("pypi.org", 443, timeout=20, context=TLS) - connection.request("GET", "/pypi/idna/json", headers={"User-Agent": USER_AGENT}) - response = connection.getresponse() - document = json.loads(response.read()) - connection.close() - wheels = [entry for entry in document["urls"] if entry["packagetype"] == "bdist_wheel"] - if not wheels: - raise RuntimeError("PyPI returned no idna wheel") - artifact = request(wheels[0]["url"], max_bytes=5_000_000) - return {"metadata": metadata, "artifact": artifact, "filename": wheels[0]["filename"]} - -def bulk_download(): - return request("https://speed.cloudflare.com/__down?bytes=5242880", max_bytes=5242880) - -def denied_destination(): - if MODE != "sandbox": - return {"skipped": "policy denial applies only to sandbox mode"} - started = time.perf_counter_ns() - try: - socket.create_connection(("example.net", 443), timeout=5).close() - except OSError as error: - return { - "denied": True, - "latency_ms": (time.perf_counter_ns() - started) / 1_000_000, - "error": str(error), - } - raise RuntimeError("destination omitted from policy was reachable") - -started = time.perf_counter_ns() -metrics = [ - metric("dns_service_set", dns_set), - metric("https_metadata_serial", metadata_serial), - metric("https_metadata_concurrent", metadata_concurrent), - metric("https_reuse", https_reuse), - metric("git_clone", git_clone), - metric("package_download", package_download), - metric("bulk_download_5mib", bulk_download), - metric("policy_denial", denied_destination), -] -document = { - "schema": "openshell.live-internet-perf.v1", - "mode": MODE, - "total_ms": (time.perf_counter_ns() - started) / 1_000_000, - "metrics": metrics, -} -print(json.dumps(document, separators=(",", ":"))) -if any(item["name"] == "policy_denial" and not item["ok"] for item in metrics): - raise SystemExit(1) -"#; - -fn rounds() -> usize { - std::env::var("OPENSHELL_LIVE_PERF_ROUNDS") - .ok() - .and_then(|value| value.parse().ok()) - .filter(|value| *value > 0) - .unwrap_or(DEFAULT_ROUNDS) -} - -fn write_policy() -> Result { - let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; - write!( - file, - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] - -landlock: - compatibility: best_effort - -network_policies: - live_internet_performance: - name: live_internet_performance - endpoints: - - {{ host: example.com, port: 443, protocol: tcp }} - - {{ host: github.com, port: 443, protocol: tcp }} - - {{ host: raw.githubusercontent.com, port: 443, protocol: tcp }} - - {{ host: pypi.org, port: 443, protocol: tcp }} - - {{ host: files.pythonhosted.org, port: 443, protocol: tcp }} - - {{ host: registry.npmjs.org, port: 443, protocol: tcp }} - - {{ host: crates.io, port: 443, protocol: tcp }} - - {{ host: docs.python.org, port: 443, protocol: tcp }} - - {{ host: speed.cloudflare.com, port: 443, protocol: tcp }} - binaries: - - path: "/**" -"# - ) - .map_err(|error| format!("write policy: {error}"))?; - file.flush() - .map_err(|error| format!("flush policy: {error}"))?; - Ok(file) -} - -async fn run_direct() -> Result { - run_python("direct").await -} - -async fn run_python(mode: &str) -> Result { - let output = tokio::process::Command::new("python3") - .args(["-c", BENCHMARK]) - .env("OPENSHELL_LIVE_PERF_MODE", mode) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .output() - .await - .map_err(|error| format!("run {mode} live Internet benchmark: {error}"))?; - let combined = format!( - "{}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - if !output.status.success() { - return Err(format!( - "{mode} live Internet benchmark failed (exit {:?}): {combined}", - output.status.code() - )); - } - Ok(combined) -} - -async fn run_sandbox(sandbox: &SandboxGuard) -> Result { - sandbox - .exec(&[ - "sh", - "-c", - "OPENSHELL_LIVE_PERF_MODE=sandbox python3 -c \"$1\"", - "openshell-live-internet-perf", - BENCHMARK, - ]) - .await -} - -#[tokio::test] -#[ignore = "manual live-Internet performance benchmark"] -async fn benchmark_live_internet_agent_traffic() { - let policy = write_policy().expect("write live Internet benchmark policy"); - let policy_path = policy.path().to_str().expect("UTF-8 policy path"); - let create_started = Instant::now(); - let mut sandbox = SandboxGuard::create(&["--policy", policy_path]) - .await - .expect("create live Internet benchmark sandbox"); - println!( - "LIVE_INTERNET_PERF create {{\"elapsed_ms\":{}}}", - create_started.elapsed().as_secs_f64() * 1000.0 - ); - - for round in 1..=rounds() { - if round % 2 == 1 { - let direct = run_direct().await.expect("direct live Internet benchmark"); - println!("LIVE_INTERNET_PERF direct round={round} {}", direct.trim()); - let mediated = run_sandbox(&sandbox) - .await - .expect("sandbox live Internet benchmark"); - println!( - "LIVE_INTERNET_PERF sandbox round={round} {}", - mediated.trim() - ); - } else { - let mediated = run_sandbox(&sandbox) - .await - .expect("sandbox live Internet benchmark"); - println!( - "LIVE_INTERNET_PERF sandbox round={round} {}", - mediated.trim() - ); - let direct = run_direct().await.expect("direct live Internet benchmark"); - println!("LIVE_INTERNET_PERF direct round={round} {}", direct.trim()); - } - } - - sandbox.cleanup().await; -} diff --git a/e2e/rust/tests/podman_oci_identity.rs b/e2e/rust/tests/podman_oci_identity.rs deleted file mode 100644 index 6b137d1a2b..0000000000 --- a/e2e/rust/tests/podman_oci_identity.rs +++ /dev/null @@ -1,213 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e-podman")] - -//! Podman-specific E2E coverage for the supervisor companion container. -//! -//! Podman runs the supervisor as the resolved OCI workload identity with no -//! effective capabilities; Docker runs it as a fixed unprivileged user. The -//! test builds an image through the selected Podman engine, creates a sandbox -//! from it, and inspects the real supervisor container. The workload's image -//! ID, identity, network, and mounts, and what the main process and -//! `sandbox exec` see, are shared with Docker and covered by the `oci-image` -//! feature suite in `tests/suites/features`. - -use std::process::Stdio; - -use openshell_e2e::harness::container::{ContainerEngine, is_e2e_driver}; -use openshell_e2e::harness::sandbox::SandboxGuard; - -const BASE_IMAGE: &str = "nvcr.io/nvidia/base/ubuntu:24.04"; -const READY_MARKER: &str = "podman-oci-identity-ready"; -const OCI_UID: &str = "2345"; -const OCI_GID: &str = "2346"; -const OCI_FALLBACK_POLICY: &str = r"version: 1 - -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /lib64, /proc, /dev/urandom, /etc] - read_write: [/sandbox, /tmp, /dev/null] -landlock: - compatibility: best_effort - -network_policies: {} -"; - -struct ImageGuard { - engine: ContainerEngine, - tag: String, -} - -impl ImageGuard { - fn build() -> Result { - let engine = ContainerEngine::from_env()?; - if engine.name() != "podman" { - return Err(format!( - "Podman OCI identity E2E requires podman, got {}", - engine.name() - )); - } - - let context = tempfile::tempdir().map_err(|err| format!("create build context: {err}"))?; - let containerfile = context.path().join("Containerfile"); - std::fs::write( - &containerfile, - format!( - r"FROM {BASE_IMAGE} -USER 0:0 -RUN mkdir -p /home/app/project && \ - chown {OCI_UID}:{OCI_GID} /home/app /home/app/project && \ - chmod 0700 /home/app /home/app/project -WORKDIR /home/app/project -USER {OCI_UID}:{OCI_GID} -" - ), - ) - .map_err(|err| format!("write Containerfile: {err}"))?; - - let tag = format!( - "localhost/openshell-e2e-podman-oci-identity:{}", - std::process::id() - ); - run_engine( - &engine, - &[ - "build", - "--pull=never", - "--file", - containerfile - .to_str() - .ok_or_else(|| "Containerfile path is not UTF-8".to_string())?, - "--tag", - &tag, - context - .path() - .to_str() - .ok_or_else(|| "build context path is not UTF-8".to_string())?, - ], - )?; - let user = run_engine( - &engine, - &["image", "inspect", "--format", "{{.Config.User}}", &tag], - )?; - if user != format!("{OCI_UID}:{OCI_GID}") { - return Err(format!( - "Podman-built image has OCI user '{user}', expected {OCI_UID}:{OCI_GID}" - )); - } - - Ok(Self { engine, tag }) - } -} - -impl Drop for ImageGuard { - fn drop(&mut self) { - let _ = self - .engine - .command() - .args(["image", "rm", "--force", &self.tag]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } -} - -fn run_engine(engine: &ContainerEngine, args: &[&str]) -> Result { - let output = engine - .command() - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .output() - .map_err(|err| format!("failed to run {} {}: {err}", engine.name(), args.join(" ")))?; - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - if !output.status.success() { - return Err(format!( - "{} {} failed (exit {:?}):\n{stdout}{stderr}", - engine.name(), - args.join(" "), - output.status.code() - )); - } - Ok(stdout.trim().to_string()) -} - -fn container_id_for_role( - engine: &ContainerEngine, - sandbox_name: &str, - role: &str, -) -> Result { - let name_filter = format!("label=openshell.ai/sandbox-name={sandbox_name}"); - let role_filter = format!("label=openshell.ai/isolation-role={role}"); - let stdout = run_engine( - engine, - &[ - "ps", - "-aq", - "--filter", - "label=openshell.managed=true", - "--filter", - &name_filter, - "--filter", - &role_filter, - ], - )?; - let ids = stdout - .lines() - .map(str::trim) - .filter(|line| !line.is_empty()) - .collect::>(); - match ids.as_slice() { - [id] => Ok((*id).to_string()), - [] => Err(format!( - "no Podman container found for sandbox '{sandbox_name}'" - )), - _ => Err(format!( - "multiple Podman containers found for sandbox '{sandbox_name}': {ids:?}" - )), - } -} - -#[tokio::test] -async fn podman_supervisor_runs_as_oci_identity_without_capabilities() { - if !is_e2e_driver("podman") { - eprintln!("Skipping Podman OCI identity test: e2e driver is not podman"); - return; - } - - let image = ImageGuard::build().expect("build Podman OCI identity image"); - // The fixture image contains a policy with an - // explicit `sandbox` process identity. Supply a complete policy that - // intentionally omits `process` so this test exercises OCI fallback. - let policy = tempfile::NamedTempFile::new().expect("create OCI fallback policy"); - std::fs::write(policy.path(), OCI_FALLBACK_POLICY).expect("write OCI fallback policy"); - let policy_path = policy.path().to_str().expect("policy path is UTF-8"); - let mut sandbox = SandboxGuard::create_keep_with_args( - &["--from", &image.tag, "--policy", policy_path, "--no-tty"], - &["sh", "-c", "echo podman-oci-identity-ready; sleep infinity"], - READY_MARKER, - ) - .await - .expect("create sandbox from Podman-built OCI identity image"); - - let supervisor_id = container_id_for_role(&image.engine, &sandbox.name, "supervisor") - .expect("find separate supervisor companion"); - let supervisor_user = run_engine( - &image.engine, - &["inspect", "--format", "{{.Config.User}}", &supervisor_id], - ) - .unwrap(); - assert_eq!(supervisor_user, format!("{OCI_UID}:{OCI_GID}")); - let supervisor_caps = run_engine( - &image.engine, - &["inspect", "--format", "{{.EffectiveCaps}}", &supervisor_id], - ) - .unwrap(); - assert_eq!( - supervisor_caps, "[]", - "the supervisor companion may not have effective capabilities" - ); - sandbox.cleanup().await; -} diff --git a/e2e/rust/tests/podman_resource_limits.rs b/e2e/rust/tests/podman_resource_limits.rs index a0282f0e09..225f77ff8f 100644 --- a/e2e/rust/tests/podman_resource_limits.rs +++ b/e2e/rust/tests/podman_resource_limits.rs @@ -7,7 +7,7 @@ //! resource limits are actually enforced by cgroups inside the workload, not //! just echoed back by the API/template layer. //! -//! `e2e/rust/tests/sandbox_templates.rs` already verifies that +//! The conformance `templates` suite already verifies that //! `sandbox template create --cpu ... --memory ...` is stored and returned //! correctly, but it never inspects the resulting container's real resource //! state. This test creates a sandbox with those flags directly and reads the diff --git a/e2e/rust/tests/sandbox_labels.rs b/e2e/rust/tests/sandbox_labels.rs deleted file mode 100644 index 937171ead3..0000000000 --- a/e2e/rust/tests/sandbox_labels.rs +++ /dev/null @@ -1,230 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use std::process::Stdio; - -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::output::{extract_field, strip_ansi}; -use openshell_e2e::harness::sandbox::SandboxGuard; - -fn normalize_output(output: &str) -> String { - let stripped = strip_ansi(output).replace('\r', ""); - let mut cleaned = String::with_capacity(stripped.len()); - - for ch in stripped.chars() { - match ch { - '\u{8}' => { - cleaned.pop(); - } - '\u{4}' => {} - _ => cleaned.push(ch), - } - } - - cleaned -} - -fn extract_sandbox_name(output: &str) -> Option { - if let Some((_, rest)) = output.split_once("Created sandbox:") { - return rest.split_whitespace().next().map(ToOwned::to_owned); - } - - extract_field(output, "Created sandbox").or_else(|| extract_field(output, "Name")) -} - -async fn create_sandbox_with_labels(name: &str, labels: &[(&str, &str)]) -> String { - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", "create", "--detach", "--name", name]); - - for (key, value) in labels { - cmd.arg("--label").arg(format!("{key}={value}")); - } - - cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell sandbox create"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = normalize_output(&format!("{stdout}{stderr}")); - - assert!( - output.status.success(), - "sandbox create should succeed (exit {:?}):\n{combined}", - output.status.code() - ); - - extract_sandbox_name(&combined).expect("sandbox name should be present in output") -} - -async fn list_sandboxes_with_selector(selector: &str) -> Vec { - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", "list", "--names"]); - - if !selector.is_empty() { - cmd.arg("--selector").arg(selector); - } - - cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell sandbox list"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = normalize_output(&format!("{stdout}{stderr}")); - - assert!( - output.status.success(), - "sandbox list should succeed (exit {:?}):\n{combined}", - output.status.code() - ); - - combined - .lines() - .map(str::trim) - .filter(|line| !line.is_empty()) - .map(ToOwned::to_owned) - .collect() -} - -async fn get_sandbox_details(name: &str) -> String { - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", "get", name]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell sandbox get"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = normalize_output(&format!("{stdout}{stderr}")); - - assert!( - output.status.success(), - "sandbox get should succeed (exit {:?}):\n{combined}", - output.status.code() - ); - - combined -} - -#[tokio::test] -#[allow(clippy::too_many_lines)] // end-to-end test exercises full label lifecycle -async fn sandbox_labels_are_stored_and_filterable() { - // Keep the unique names below the public 19-character routable-name - // limit. Hex keeps the per-process suffix compact on busy CI hosts. - let suffix = format!("{:x}", std::process::id()); - let dev_backend = format!("lbl-db-{suffix}"); - let staging_backend = format!("lbl-sb-{suffix}"); - let prod_frontend = format!("lbl-pf-{suffix}"); - let dev_data = format!("lbl-dd-{suffix}"); - - // Arm guards before create so a partial create still cleans up. - let _cleanup: Vec<_> = [&dev_backend, &staging_backend, &prod_frontend, &dev_data] - .into_iter() - .map(|name| SandboxGuard::manage_existing(name.clone())) - .collect(); - - // Create sandboxes with different labels - let name1 = - create_sandbox_with_labels(&dev_backend, &[("env", "dev"), ("team", "backend")]).await; - - let name2 = - create_sandbox_with_labels(&staging_backend, &[("env", "staging"), ("team", "backend")]) - .await; - - let name3 = - create_sandbox_with_labels(&prod_frontend, &[("env", "prod"), ("team", "frontend")]).await; - - let name4 = create_sandbox_with_labels(&dev_data, &[("env", "dev"), ("team", "data")]).await; - - // Test 1: Verify labels are stored in sandbox metadata - let details = get_sandbox_details(&name1).await; - assert!( - details.contains("env: dev"), - "sandbox should have env=dev label in details:\n{details}" - ); - assert!( - details.contains("team: backend"), - "sandbox should have team=backend label in details:\n{details}" - ); - - // Test 2: Filter by single label (env=dev) - let dev_sandboxes = list_sandboxes_with_selector("env=dev").await; - assert!( - dev_sandboxes.contains(&name1), - "env=dev filter should include {name1}, got: {dev_sandboxes:?}" - ); - assert!( - dev_sandboxes.contains(&name4), - "env=dev filter should include {name4}, got: {dev_sandboxes:?}" - ); - assert!( - !dev_sandboxes.contains(&name2), - "env=dev filter should not include staging sandbox {name2}, got: {dev_sandboxes:?}" - ); - assert!( - !dev_sandboxes.contains(&name3), - "env=dev filter should not include prod sandbox {name3}, got: {dev_sandboxes:?}" - ); - - // Test 3: Filter by single label (team=backend) - let backend_sandboxes = list_sandboxes_with_selector("team=backend").await; - assert!( - backend_sandboxes.contains(&name1), - "team=backend filter should include {name1}, got: {backend_sandboxes:?}" - ); - assert!( - backend_sandboxes.contains(&name2), - "team=backend filter should include {name2}, got: {backend_sandboxes:?}" - ); - assert!( - !backend_sandboxes.contains(&name3), - "team=backend filter should not include frontend sandbox {name3}, got: {backend_sandboxes:?}" - ); - assert!( - !backend_sandboxes.contains(&name4), - "team=backend filter should not include data sandbox {name4}, got: {backend_sandboxes:?}" - ); - - // Test 4: Filter by multiple labels (AND logic: env=dev,team=backend) - let dev_backend_sandboxes = list_sandboxes_with_selector("env=dev,team=backend").await; - assert_eq!( - dev_backend_sandboxes - .iter() - .filter(|name| [&name1, &name2, &name3, &name4].contains(name)) - .count(), - 1, - "env=dev,team=backend filter should return exactly 1 sandbox, got: {dev_backend_sandboxes:?}" - ); - assert!( - dev_backend_sandboxes.contains(&name1), - "env=dev,team=backend filter should include {name1}, got: {dev_backend_sandboxes:?}" - ); - - // Test 5: Filter by non-existent label value - let qa_sandboxes = list_sandboxes_with_selector("team=qa").await; - assert!( - !qa_sandboxes.contains(&name1) - && !qa_sandboxes.contains(&name2) - && !qa_sandboxes.contains(&name3) - && !qa_sandboxes.contains(&name4), - "team=qa filter should not return any of our test sandboxes, got: {qa_sandboxes:?}" - ); - - // Test 6: List all sandboxes (no filter) - let all_sandboxes = list_sandboxes_with_selector("").await; - assert!( - all_sandboxes.contains(&name1), - "list without filter should include all test sandboxes" - ); - assert!( - all_sandboxes.contains(&name2), - "list without filter should include all test sandboxes" - ); - assert!( - all_sandboxes.contains(&name3), - "list without filter should include all test sandboxes" - ); - assert!( - all_sandboxes.contains(&name4), - "list without filter should include all test sandboxes" - ); -} diff --git a/e2e/rust/tests/sandbox_lifecycle.rs b/e2e/rust/tests/sandbox_lifecycle.rs index b299a5d1d7..748ea40137 100644 --- a/e2e/rust/tests/sandbox_lifecycle.rs +++ b/e2e/rust/tests/sandbox_lifecycle.rs @@ -9,9 +9,7 @@ use std::process::Stdio; use std::time::Duration; use openshell_e2e::harness::binary::{openshell_cmd, openshell_tty_cmd}; -use openshell_e2e::harness::cli::{ - run_cli, wait_for_sandbox_exec_contains, wait_for_sandbox_phase, -}; +use openshell_e2e::harness::cli::{run_cli, wait_for_sandbox_exec_contains}; use openshell_e2e::harness::output::{extract_field, strip_ansi}; use openshell_e2e::harness::sandbox::{E2E_WORKLOAD_IMAGE, SandboxGuard, unique_sandbox_name}; use serial_test::serial; @@ -114,28 +112,6 @@ async fn delete_sandbox(name: &str) { let _ = cmd.status().await; } -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn sandbox_exec_outlives_startup_recovery_deadline() { - let mut sandbox = SandboxGuard::create(&[]) - .await - .expect("create sandbox for exec recovery deadline"); - let output = tokio::time::timeout( - Duration::from_secs(60), - sandbox.exec(&["sh", "-c", "sleep 31; printf past-recovery-deadline"]), - ) - .await; - let next = sandbox.exec(&["printf", "fresh-exec"]).await; - sandbox.cleanup().await; - - assert_eq!( - output.expect("exec timed out").expect("exec failed"), - "past-recovery-deadline", - "expiration must not terminate an admitted command" - ); - assert_eq!(next.expect("fresh exec after expiration"), "fresh-exec"); -} - #[tokio::test] #[serial(sandbox_lifecycle)] async fn sandbox_exec_large_output_is_complete() { @@ -387,39 +363,6 @@ async fn piped_exec_stdin_crosses_grpc_message_limit() { sandbox.cleanup().await; } -async fn run_sandbox_lifecycle_command(operation: &str, name: &str) -> String { - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", operation, name]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let output = cmd - .output() - .await - .unwrap_or_else(|error| panic!("spawn openshell sandbox {operation}: {error}")); - let combined = normalize_output(&format!( - "{}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr), - )); - assert!( - output.status.success(), - "sandbox {operation} should succeed (exit {:?}):\n{combined}", - output.status.code(), - ); - combined -} - -async fn sandbox_details(name: &str) -> String { - let (details, exit_code) = run_cli(&["sandbox", "get", name]).await; - let details = normalize_output(&details); - assert_eq!( - exit_code, 0, - "sandbox get should succeed for {name}:\n{details}" - ); - details -} - async fn reconnect_with_input_ownership( sandbox_name: &str, ) -> (tokio::process::Child, Vec) { @@ -547,125 +490,6 @@ async fn wait_for_process_with_args(expected_args: &[&str]) -> u32 { .unwrap_or_else(|_| panic!("process with arguments {expected_args:?} did not start")) } -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn sandbox_stop_start_preserves_workspace() { - const SENTINEL: &str = "openshell-stop-start-sentinel"; - const SENTINEL_PATH: &str = "/sandbox/.openshell-stop-start-e2e"; - const RUN_COUNT_PATH: &str = "/sandbox/.openshell-main-run-count"; - let write_sentinel = format!("printf '%s\\n' '{SENTINEL}' > '{SENTINEL_PATH}'"); - let main = format!( - "count=0; test ! -f '{RUN_COUNT_PATH}' || count=$(cat '{RUN_COUNT_PATH}'); \ - count=$((count + 1)); printf '%s\\n' \"$count\" > '{RUN_COUNT_PATH}'; \ - echo lifecycle-ready-$count; exec sleep infinity" - ); - - let mut sandbox = SandboxGuard::create_keep(&["sh", "-c", &main], "lifecycle-ready") - .await - .expect("sandbox create should start a durable main process"); - sandbox - .exec(&["sh", "-lc", &write_sentinel]) - .await - .expect("sandbox exec should write the workspace sentinel"); - - let stop_output = run_sandbox_lifecycle_command("stop", &sandbox.name).await; - assert!( - stop_output.contains("Stopped sandbox"), - "expected stop confirmation in:\n{stop_output}", - ); - - let mut exec_cmd = openshell_cmd(); - exec_cmd - .args([ - "sandbox", - "exec", - "--name", - &sandbox.name, - "--no-tty", - "--", - "cat", - SENTINEL_PATH, - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let stopped_exec = exec_cmd - .output() - .await - .expect("spawn openshell sandbox exec while stopped"); - assert!( - !stopped_exec.status.success(), - "sandbox exec should fail while stopped" - ); - - let start_output = run_sandbox_lifecycle_command("start", &sandbox.name).await; - assert!( - start_output.contains("Started sandbox"), - "expected start confirmation in:\n{start_output}", - ); - - let sentinel = sandbox - .exec(&["cat", SENTINEL_PATH]) - .await - .expect("sandbox exec should succeed after start"); - assert!( - sentinel.lines().any(|line| line.trim() == SENTINEL), - "workspace sentinel should survive stop and start:\n{sentinel}", - ); - let run_count = sandbox - .exec(&["cat", RUN_COUNT_PATH]) - .await - .expect("sandbox exec should read the canonical main run count"); - assert_eq!( - run_count.trim(), - "2", - "start should launch a fresh canonical main instance:\n{run_count}", - ); - - sandbox.cleanup().await; -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn sandbox_can_be_deleted_while_stopped() { - let mut sandbox = SandboxGuard::create_keep( - &["sh", "-c", "echo stop-ready; exec sleep infinity"], - "stop-ready", - ) - .await - .expect("sandbox create should start a durable main process"); - - let stop_output = run_sandbox_lifecycle_command("stop", &sandbox.name).await; - assert!( - stop_output.contains("Stopped sandbox"), - "expected stop confirmation in:\n{stop_output}", - ); - - let delete_output = run_sandbox_lifecycle_command("delete", &sandbox.name).await; - // Deletion may return before the owned cleanup worker finishes. Both - // outcomes must still reach absence, which is checked below. - assert!( - delete_output.contains(&format!("Deleted sandbox {}", sandbox.name)) - || delete_output.contains(&format!( - "Sandbox {} deletion accepted; cleanup is pending", - sandbox.name - )), - "expected completed or accepted deletion in:\n{delete_output}", - ); - - if let Err(last_sandbox_list) = assert_sandbox_presence_eventually(&sandbox.name, false).await { - sandbox.cleanup().await; - panic!( - "stopped sandbox {} should be deleted without starting after \ - {SANDBOX_PRESENCE_TIMEOUT:?}; last observed sandbox list: {last_sandbox_list:?}", - sandbox.name, - ); - } - - // Mark the guard cleaned up. Its idempotent delete is harmless now that - // the lifecycle operation above has removed the sandbox. - sandbox.cleanup().await; -} - #[tokio::test] #[serial(sandbox_lifecycle)] async fn canonical_main_exit_zero_completes_persistent_sandbox() { @@ -779,166 +603,6 @@ async fn canonical_main_nonzero_exit_preserves_status() { ); } -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn detached_canonical_main_exit_zero_reaches_completed() { - const RELEASE_PATH: &str = "/sandbox/.openshell-detached-success-release"; - let script = format!("while [ ! -e '{RELEASE_PATH}' ]; do sleep 0.05; done; exit 0"); - let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-c", &script]) - .await - .expect("create detached successful canonical main"); - sandbox - .exec(&["touch", RELEASE_PATH]) - .await - .expect("release detached successful canonical main"); - - wait_for_sandbox_phase(&sandbox.name, "Completed", SANDBOX_PRESENCE_TIMEOUT) - .await - .unwrap_or_else(|err| panic!("detached successful main did not complete:\n{err}")); - let details = sandbox_details(&sandbox.name).await; - assert!( - details.contains("Phase: Completed"), - "detached successful main should retain Completed status:\n{details}" - ); - - sandbox.cleanup().await; -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn detached_canonical_main_nonzero_exit_reaches_error() { - const RELEASE_PATH: &str = "/sandbox/.openshell-detached-failure-release"; - let script = format!("while [ ! -e '{RELEASE_PATH}' ]; do sleep 0.05; done; exit 11"); - let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-c", &script]) - .await - .expect("create detached failing canonical main"); - sandbox - .exec(&["touch", RELEASE_PATH]) - .await - .expect("release detached failing canonical main"); - - wait_for_sandbox_phase(&sandbox.name, "Error", SANDBOX_PRESENCE_TIMEOUT) - .await - .unwrap_or_else(|err| panic!("detached failing main did not reach Error:\n{err}")); - let details = sandbox_details(&sandbox.name).await; - assert!( - details.contains("Phase: Error") && details.contains("Exit Code: 11"), - "detached failing main should retain its terminal result:\n{details}" - ); - - sandbox.cleanup().await; -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn canonical_main_and_exec_receive_declared_environment() { - for mode in ["--tty", "--no-tty"] { - let script = r#"printf 'declared_env=%s\n' "${REPRO_SENTINEL:-missing}"; while true; do sleep 1; done"#; - let mut sandbox = SandboxGuard::create_keep_with_args( - &[ - mode, - "--no-auto-providers", - "--env", - "REPRO_SENTINEL=present", - ], - &["sh", "-c", script], - "declared_env=", - ) - .await - .expect("create canonical process with declared environment"); - let initial = normalize_output(&sandbox.create_output); - let later = sandbox - .exec(&[ - "sh", - "-c", - r#"printf 'declared_env=%s\n' "${REPRO_SENTINEL:-missing}""#, - ]) - .await; - sandbox.cleanup().await; - - assert!( - initial.lines().any(|line| line == "declared_env=present"), - "initial process must receive declared environment ({mode}): {initial}" - ); - let later = normalize_output(&later.expect("exec environment probe")); - assert!( - later.lines().any(|line| line == "declared_env=present"), - "exec must receive the same declared environment ({mode}): {later}" - ); - } -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn detached_main_exit_during_provisioning_is_classified_as_workload_result() { - let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-c", "exit 11"]) - .await - .expect("fast detached main exit should not be reported as a provisioning failure"); - - wait_for_sandbox_phase(&sandbox.name, "Error", SANDBOX_PRESENCE_TIMEOUT) - .await - .unwrap_or_else(|err| panic!("fast detached main did not reach Error:\n{err}")); - - let mut get_cmd = openshell_cmd(); - get_cmd - .args(["sandbox", "get", &sandbox.name]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let get_output = get_cmd.output().await.expect("spawn openshell sandbox get"); - let details = normalize_output(&format!( - "{}{}", - String::from_utf8_lossy(&get_output.stdout), - String::from_utf8_lossy(&get_output.stderr), - )); - assert!( - get_output.status.success(), - "sandbox get failed:\n{details}" - ); - assert!( - details.contains("Phase: Error") && details.contains("Exit Code: 11"), - "fast detached main should retain its workload result:\n{details}" - ); - - sandbox.cleanup().await; -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn canonical_tty_main_uses_sandbox_environment() { - let script = r#"printf 'canonical_env home=%s user=%s term=%s\n' "$HOME" "$USER" "$TERM"; while true; do sleep 1; done"#; - let mut sandbox = - SandboxGuard::create_keep_with_args(&["--tty"], &["sh", "-lc", script], "canonical_env") - .await - .expect("create canonical TTY process"); - - let output = normalize_output(&sandbox.create_output); - let environment = output - .lines() - .find(|line| line.contains("canonical_env")) - .expect("canonical environment output"); - let field = |name: &str| { - environment - .split_whitespace() - .find_map(|value| value.strip_prefix(&format!("{name}="))) - .unwrap_or_default() - }; - - assert!( - !field("home").is_empty() && field("home") != "/root", - "canonical process must not inherit the supervisor HOME: {environment}" - ); - assert!( - !field("user").is_empty(), - "canonical process USER must identify the sandbox user: {environment}" - ); - assert!( - !field("term").is_empty() && field("term") != "dumb", - "canonical TTY process must receive a usable TERM: {environment}" - ); - - sandbox.cleanup().await; -} - #[tokio::test] #[serial(sandbox_lifecycle)] async fn canonical_main_disconnect_reconnect_replays_history_for_same_process() { @@ -1299,95 +963,6 @@ async fn canonical_main_exit_255_is_not_retried_as_transport_failure() { assert_eq!(status.code(), Some(255)); } -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn on_failure_policy_replaces_runtime_and_preserves_workspace() { - const SCRIPT: &str = r#" -marker=/sandbox/.openshell-restart-e2e -if [ -e "$marker" ]; then - printf 'replacement-%s\n' "$(cat /proc/sys/kernel/random/uuid)" > /sandbox/replacement-run - sleep 300 -else - touch "$marker" - printf 'initial-%s\n' "$(cat /proc/sys/kernel/random/uuid)" > /sandbox/initial-run - while [ ! -e /sandbox/.openshell-restart-release ]; do sleep 0.05; done - exit 17 -fi -"#; - - let mut sandbox = SandboxGuard::manage_existing(unique_sandbox_name()); - let (output, exit_code) = run_cli(&[ - "sandbox", - "create", - "--name", - &sandbox.name, - "--from", - E2E_WORKLOAD_IMAGE, - "--detach", - "--restart-policy", - "on-failure", - "--", - "sh", - "-lc", - SCRIPT, - ]) - .await; - assert_eq!( - exit_code, 0, - "create sandbox with OnFailure policy: {output}" - ); - wait_for_sandbox_exec_contains( - &sandbox.name, - &["cat", "/sandbox/initial-run"], - "initial-", - SANDBOX_RESTART_TIMEOUT, - ) - .await - .expect("initial main process should write its workspace marker"); - - let initial_details = sandbox_details(&sandbox.name).await; - let initial_instance = extract_field(&initial_details, "Main process instance") - .expect("initial main process instance"); - sandbox - .exec(&["touch", "/sandbox/.openshell-restart-release"]) - .await - .expect("release initial main process"); - - let deadline = Instant::now() + SANDBOX_RESTART_TIMEOUT; - let mut last_runs = Err("replacement is not ready".to_string()); - let final_details = loop { - let details = sandbox_details(&sandbox.name).await; - let replacement_instance = extract_field(&details, "Main process instance") - .is_some_and(|instance| instance != "-" && instance != initial_instance); - if details.contains("Phase: Ready") - && extract_field(&details, "Restart count").as_deref() == Some("1") - && replacement_instance - { - last_runs = sandbox - .exec(&["cat", "/sandbox/initial-run", "/sandbox/replacement-run"]) - .await; - if let Ok(runs) = &last_runs - && runs.contains("initial-") - && runs.contains("replacement-") - { - break details; - } - } - assert!( - Instant::now() < deadline, - "sandbox did not complete its policy-driven restart within \ - {SANDBOX_RESTART_TIMEOUT:?}; last details:\n{details}\nworkspace: {last_runs:?}" - ); - sleep(Duration::from_millis(250)).await; - }; - - assert!( - final_details.contains("Restart policy: on-failure"), - "restart policy should remain visible after replacement:\n{final_details}" - ); - sandbox.cleanup().await; -} - #[tokio::test] async fn sandbox_create_with_no_keep_cleans_up_after_tty_command() { let name = format!("tty-{:015x}", rand::random::() & 0x0fff_ffff_ffff_ffff); @@ -1513,3 +1088,105 @@ async fn sandbox_create_with_no_keep_preserves_failure_then_cleans_up() { ); } } + +async fn sandbox_details(name: &str) -> String { + let (details, exit_code) = run_cli(&["sandbox", "get", name]).await; + let details = normalize_output(&details); + assert_eq!( + exit_code, 0, + "sandbox get should succeed for {name}:\n{details}" + ); + details +} + +// The conformance suite has no counterpart for this test: on the VM driver the +// replacement never starts (https://github.com/NVIDIA/OpenShell/issues/5507), +// and the legacy lanes that run this file are Docker and Kubernetes. +#[tokio::test] +#[serial(sandbox_lifecycle)] +async fn on_failure_policy_replaces_runtime_and_preserves_workspace() { + const SCRIPT: &str = r#" +marker=/sandbox/.openshell-restart-e2e +if [ -e "$marker" ]; then + printf 'replacement-%s\n' "$(cat /proc/sys/kernel/random/uuid)" > /sandbox/replacement-run + sleep 300 +else + touch "$marker" + printf 'initial-%s\n' "$(cat /proc/sys/kernel/random/uuid)" > /sandbox/initial-run + while [ ! -e /sandbox/.openshell-restart-release ]; do sleep 0.05; done + exit 17 +fi +"#; + + let mut sandbox = SandboxGuard::manage_existing(unique_sandbox_name()); + let (output, exit_code) = run_cli(&[ + "sandbox", + "create", + "--name", + &sandbox.name, + "--from", + E2E_WORKLOAD_IMAGE, + "--detach", + "--restart-policy", + "on-failure", + "--", + "sh", + "-lc", + SCRIPT, + ]) + .await; + assert_eq!( + exit_code, 0, + "create sandbox with OnFailure policy: {output}" + ); + wait_for_sandbox_exec_contains( + &sandbox.name, + &["cat", "/sandbox/initial-run"], + "initial-", + SANDBOX_RESTART_TIMEOUT, + ) + .await + .expect("initial main process should write its workspace marker"); + + let initial_details = sandbox_details(&sandbox.name).await; + let initial_instance = extract_field(&initial_details, "Main process instance") + .expect("initial main process instance"); + sandbox + .exec(&["touch", "/sandbox/.openshell-restart-release"]) + .await + .expect("release initial main process"); + + let deadline = Instant::now() + SANDBOX_RESTART_TIMEOUT; + let mut last_runs = Err("replacement is not ready".to_string()); + let final_details = loop { + let details = sandbox_details(&sandbox.name).await; + let replacement_instance = extract_field(&details, "Main process instance") + .is_some_and(|instance| instance != "-" && instance != initial_instance); + if details.contains("Phase: Ready") + && extract_field(&details, "Restart count").as_deref() == Some("1") + && replacement_instance + { + last_runs = sandbox + .exec(&["cat", "/sandbox/initial-run", "/sandbox/replacement-run"]) + .await; + if let Ok(runs) = &last_runs + && runs.contains("initial-") + && runs.contains("replacement-") + { + break details; + } + } + assert!( + Instant::now() < deadline, + "sandbox did not complete its policy-driven restart within \ + {SANDBOX_RESTART_TIMEOUT:?}; last details:\n{details}\nworkspace: {last_runs:?}" + ); + sleep(Duration::from_millis(250)).await; + }; + + assert!( + final_details.contains("Restart policy: on-failure"), + "restart policy should remain visible after replacement:\n{final_details}" + ); + sandbox.cleanup().await; +} diff --git a/e2e/rust/tests/sandbox_templates.rs b/e2e/rust/tests/sandbox_templates.rs deleted file mode 100644 index 21c9233daf..0000000000 --- a/e2e/rust/tests/sandbox_templates.rs +++ /dev/null @@ -1,275 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E coverage for reusable sandbox workload templates. - -use std::process::Stdio; -use std::time::{SystemTime, UNIX_EPOCH}; - -use openshell_e2e::harness::binary::{openshell_bin, openshell_cmd}; -use openshell_e2e::harness::output::strip_ansi; -use openshell_e2e::harness::sandbox::SandboxGuard; -use serde_json::Value; - -struct CliResult { - output: String, - success: bool, -} - -struct TemplateGuard { - name: String, -} - -impl TemplateGuard { - fn new(name: String) -> Self { - Self { name } - } - - async fn cleanup(mut self) { - delete_template(&self.name).await; - self.name.clear(); - } -} - -impl Drop for TemplateGuard { - fn drop(&mut self) { - if self.name.is_empty() { - return; - } - let bin = openshell_bin(); - let _ = std::process::Command::new(&bin) - .args(["sandbox", "template", "delete", &self.name]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } -} - -async fn run_cli(args: &[&str]) -> CliResult { - let mut cmd = openshell_cmd(); - cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell command"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - - CliResult { - output: strip_ansi(&combined), - success: output.status.success(), - } -} - -async fn delete_template(name: &str) { - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", "template", "delete", name]) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - -fn unique_name(prefix: &str) -> String { - let millis = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system clock before epoch") - .as_millis(); - let suffix = millis % 1_000_000; - format!("{prefix}-{suffix:06}") -} - -fn output_contains(output: &str, needle: &str) -> bool { - output.to_lowercase().contains(&needle.to_lowercase()) -} - -fn output_mentions_template_not_found(output: &str) -> bool { - output_contains(output, "sandbox template") && output_contains(output, "not found") -} - -#[tokio::test] -async fn sandbox_create_from_template_uses_reusable_workload() { - let template_name = unique_name("tmpl"); - let sandbox_name = unique_name("sb-tmpl"); - let template = TemplateGuard::new(template_name.clone()); - - let create_template = run_cli(&[ - "sandbox", - "template", - "create", - &template_name, - "--cpu", - "500m", - "--memory", - "512Mi", - "--label", - "e2e=sandbox-template", - "--env", - "FEATURE_FLAG=on", - ]) - .await; - assert!( - create_template.success, - "sandbox template create failed:\n{}", - create_template.output - ); - - let get_template = run_cli(&[ - "sandbox", - "template", - "get", - &template_name, - "--output", - "json", - ]) - .await; - assert!( - get_template.success, - "sandbox template get failed:\n{}", - get_template.output - ); - let template_json: Value = - serde_json::from_str(&get_template.output).expect("template JSON output"); - assert_eq!(template_json["name"].as_str(), Some(template_name.as_str())); - assert_eq!( - template_json["labels"]["e2e"].as_str(), - Some("sandbox-template") - ); - assert_eq!( - template_json["environment"]["FEATURE_FLAG"].as_str(), - Some("on") - ); - assert_eq!(template_json["resources"]["cpu"].as_str(), Some("500m")); - assert_eq!(template_json["resources"]["memory"].as_str(), Some("512Mi")); - - let list_templates = run_cli(&["sandbox", "template", "list", "--names"]).await; - assert!( - list_templates.success, - "sandbox template list failed:\n{}", - list_templates.output - ); - assert!( - list_templates - .output - .lines() - .any(|line| line.trim() == template_name), - "template list should include {template_name}:\n{}", - list_templates.output - ); - - let mut sandbox = SandboxGuard::create(&[ - "--name", - &sandbox_name, - "--template", - &template_name, - "--", - "sh", - "-lc", - "test \"$FEATURE_FLAG\" = on && echo template-env-ok", - ]) - .await - .expect("sandbox create from template should succeed"); - - assert!( - sandbox.create_output.contains("template-env-ok"), - "sandbox should inherit template environment:\n{}", - sandbox.create_output - ); - - sandbox.cleanup().await; - template.cleanup().await; -} - -#[tokio::test] -async fn sandbox_template_get_after_delete_returns_not_found() { - let template_name = unique_name("tmpl-del"); - let template = TemplateGuard::new(template_name.clone()); - - let create_template = run_cli(&["sandbox", "template", "create", &template_name]).await; - assert!( - create_template.success, - "sandbox template create failed:\n{}", - create_template.output - ); - - let delete_template = run_cli(&["sandbox", "template", "delete", &template_name]).await; - assert!( - delete_template.success, - "sandbox template delete failed:\n{}", - delete_template.output - ); - - let get_deleted = run_cli(&["sandbox", "template", "get", &template_name]).await; - assert!( - !get_deleted.success, - "deleted template should not be fetchable:\n{}", - get_deleted.output - ); - assert!( - output_mentions_template_not_found(&get_deleted.output), - "deleted template should return not-found:\n{}", - get_deleted.output - ); - - template.cleanup().await; -} - -#[tokio::test] -async fn sandbox_template_create_rejects_duplicate_name() { - let template_name = unique_name("tmpl-dupe"); - let template = TemplateGuard::new(template_name.clone()); - - let create_template = run_cli(&["sandbox", "template", "create", &template_name]).await; - assert!( - create_template.success, - "sandbox template create failed:\n{}", - create_template.output - ); - - let duplicate = run_cli(&["sandbox", "template", "create", &template_name]).await; - assert!( - !duplicate.success, - "duplicate sandbox template create should fail:\n{}", - duplicate.output - ); - assert!( - output_contains(&duplicate.output, "already exists"), - "duplicate sandbox template create should report already-exists:\n{}", - duplicate.output - ); - - template.cleanup().await; -} - -#[tokio::test] -async fn sandbox_create_from_missing_template_returns_not_found() { - let template_name = unique_name("tmpl-missing"); - let sandbox_name = unique_name("sb-missing"); - - let create_sandbox = run_cli(&[ - "sandbox", - "create", - "--name", - &sandbox_name, - "--template", - &template_name, - "--", - "true", - ]) - .await; - - if create_sandbox.success { - let _ = run_cli(&["sandbox", "delete", &sandbox_name]).await; - } - - assert!( - !create_sandbox.success, - "sandbox create from missing template should fail:\n{}", - create_sandbox.output - ); - assert!( - output_mentions_template_not_found(&create_sandbox.output), - "sandbox create from missing template should report not-found:\n{}", - create_sandbox.output - ); -} diff --git a/e2e/rust/tests/upload_backpressure.rs b/e2e/rust/tests/upload_backpressure.rs deleted file mode 100644 index e766303c37..0000000000 --- a/e2e/rust/tests/upload_backpressure.rs +++ /dev/null @@ -1,121 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(all(feature = "e2e", unix))] - -//! Regression for large built-in uploads through the CLI, gateway, supervisor, -//! and sandbox. A fast sender previously overflowed the supervisor's 4 MiB -//! pending-stdin budget while tar was creating many files (issue #4430). - -use std::fmt::Write as _; -use std::fs; -use std::os::unix::fs::{PermissionsExt, symlink}; -use std::time::Duration; - -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::output::strip_ansi; -use openshell_e2e::harness::sandbox::SandboxGuard; -use rand::{RngCore, SeedableRng, rngs::StdRng}; -use sha2::{Digest, Sha256}; -use tokio::time::timeout; - -const FILE_COUNT: usize = 12_000; -const FILE_BYTES: usize = 8_192; -const TRANSFER_TIMEOUT: Duration = Duration::from_secs(300); - -/// An archive substantially larger than the stdin budget must finish without -/// pacing at the caller, preserving every file's content and tar semantics. -#[tokio::test] -async fn large_upload_preserves_content_and_metadata_outside_workdir() { - let tmpdir = tempfile::tempdir().expect("create upload fixture tempdir"); - let source = tmpdir.path().join("large-project"); - fs::create_dir_all(source.join("payload")).expect("create payload directory"); - - // Distinct deterministic pseudorandom bytes keep the fixture reproducible - // without letting compression or identical-file checks hide dropped data. - let mut rng = StdRng::seed_from_u64(4430); - let mut payload = [0_u8; FILE_BYTES]; - let mut checksums = String::new(); - for index in 0..FILE_COUNT { - rng.fill_bytes(&mut payload); - let relative = format!("payload/{index:05}.bin"); - fs::write(source.join(&relative), payload).expect("write payload file"); - writeln!(checksums, "{:x} {relative}", Sha256::digest(payload)) - .expect("append file checksum"); - } - let manifest_digest = format!("{:x}", Sha256::digest(checksums.as_bytes())); - fs::write(source.join("SHA256SUMS"), checksums).expect("write checksum manifest"); - fs::write( - source.join("run.sh"), - "#!/bin/sh\nprintf '%s\\n' upload-executable-ok\n", - ) - .expect("write executable fixture"); - fs::set_permissions(source.join("run.sh"), fs::Permissions::from_mode(0o751)) - .expect("set executable mode"); - symlink("payload/00000.bin", source.join("payload-link")).expect("create relative symlink"); - symlink("missing-target", source.join("dangling-link")).expect("create dangling symlink"); - fs::write(source.join("conflict.txt"), "local-conflict").expect("write merge fixture"); - - let mut sandbox = SandboxGuard::create(&[]) - .await - .expect("create persistent upload sandbox"); - sandbox - .exec(&[ - "sh", - "-c", - "set -eu; test \"$(pwd -P)\" != /tmp/openshell-upload-backpressure; \ - mkdir -p /tmp/openshell-upload-backpressure/large-project; \ - printf remote-conflict > /tmp/openshell-upload-backpressure/large-project/conflict.txt; \ - printf remote-preserved > /tmp/openshell-upload-backpressure/large-project/unrelated.txt", - ]) - .await - .expect("prepare destination outside the canonical workdir"); - - // No sleep, bandwidth limit, or retry: the protocol must backpressure the - // normal CLI tar stream, even when the local sender outruns extraction. - let mut upload = openshell_cmd(); - upload - .env("RUST_LOG", "warn") - .args(["sandbox", "upload", &sandbox.name]) - .arg(&source) - .args(["/tmp/openshell-upload-backpressure", "--no-git-ignore"]); - let result = timeout(TRANSFER_TIMEOUT, upload.output()) - .await - .expect("large upload must not hang waiting for credit") - .expect("start large upload"); - let output = format!( - "{}{}", - String::from_utf8_lossy(&result.stdout), - String::from_utf8_lossy(&result.stderr) - ); - assert!(result.status.success(), "large upload failed: {output}"); - assert!( - !output.contains("sandbox sync operation failed; retrying"), - "large upload required a retry: {output}" - ); - - let verify = format!( - "set -eu; cd /tmp/openshell-upload-backpressure/large-project; \ - test \"$(find payload -type f | wc -l)\" -eq {FILE_COUNT}; \ - test \"$(sha256sum SHA256SUMS | cut -d ' ' -f 1)\" = {manifest_digest}; \ - sha256sum --check --quiet SHA256SUMS; \ - test \"$(stat -c %a run.sh)\" = 751; \ - test \"$(./run.sh)\" = upload-executable-ok; \ - test -L payload-link; test \"$(readlink payload-link)\" = payload/00000.bin; \ - test -L dangling-link; test ! -e dangling-link; \ - test \"$(readlink dangling-link)\" = missing-target; \ - test \"$(cat conflict.txt)\" = local-conflict; \ - test \"$(cat unrelated.txt)\" = remote-preserved; \ - printf large-upload-content-and-metadata-ok" - ); - let output = timeout(TRANSFER_TIMEOUT, sandbox.exec(&["sh", "-c", &verify])) - .await - .expect("large upload verification must not hang") - .expect("every uploaded file and metadata item must match"); - assert!( - strip_ansi(&output).contains("large-upload-content-and-metadata-ok"), - "expected complete upload verification marker:\n{output}" - ); - - sandbox.cleanup().await; -} diff --git a/e2e/rust/tests/vm_corporate_proxy.rs b/e2e/rust/tests/vm_corporate_proxy.rs index d914cb1c03..32d9934125 100644 --- a/e2e/rust/tests/vm_corporate_proxy.rs +++ b/e2e/rust/tests/vm_corporate_proxy.rs @@ -21,8 +21,6 @@ //! 407 to an unauthenticated CONNECT, so a 200 proves delivery. //! 4. A port-qualified `no_proxy` entry bypasses the proxy for that port only. //! 5. An `https://` proxy works when its CA is supplied via `proxy_ca_bundle`. -//! 6. An incoherent setting is fatal at gateway startup rather than -//! degrading to a direct dial. //! //! Fixtures run as host processes and are reached by the host supervisor. //! `host.openshell.internal` is normalized to host loopback for both libkrun @@ -850,39 +848,3 @@ async fn vm_corporate_proxy_trusts_ca_bundle_for_https_proxy() { .await .expect("restore gateway config"); } - -#[tokio::test] -#[serial(vm_corporate_proxy)] -async fn vm_corporate_proxy_rejects_incoherent_configuration() { - if !should_run("corporate proxy fail-closed test") { - return; - } - - // A bypass list without a proxy URL means the operator believed proxying - // was in effect. Accepting it would leave every dial direct while looking - // configured, so the gateway must refuse to start instead. - let mut gateway_config = GatewayProxyConfig::apply_raw("no_proxy = \"10.0.0.0/8\"\n") - .expect("apply incoherent corporate proxy gateway config"); - - let health = wait_for_healthy(Duration::from_secs(20)).await; - assert!( - health.is_err(), - "gateway must not serve traffic with an incoherent [openshell.drivers.vm] proxy table" - ); - - let log_path = std::env::var("OPENSHELL_E2E_GATEWAY_LOG").expect("gateway log path"); - let log = std::fs::read_to_string(&log_path).expect("read gateway log"); - // The message must name the offending key rather than surfacing as an - // opaque driver-readiness timeout. Matched loosely on the key names so - // this does not break on error-wrapping changes. - assert!( - log.contains("no_proxy") && log.contains("https_proxy"), - "the startup error must name the offending key; gateway log tail:\n{}", - log.lines().rev().take(40).collect::>().join("\n") - ); - - gateway_config - .restore() - .await - .expect("restore gateway config"); -} diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index aca4d60987..855f0ccac8 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -517,6 +517,7 @@ name = "conformance" version = "0.0.0" dependencies = [ "host-server-support", + "openshell-crypto", "openshell-testkit", "serde_json", "tempfile", diff --git a/tests/suites/conformance/Cargo.toml b/tests/suites/conformance/Cargo.toml index 7201953c9d..9715526af2 100644 --- a/tests/suites/conformance/Cargo.toml +++ b/tests/suites/conformance/Cargo.toml @@ -22,6 +22,7 @@ test = false [dependencies] host-server-support = { path = "../../support/host-server" } +openshell-crypto = { path = "../../../crates/openshell-crypto" } openshell-testkit = { path = "../../testkit/testkit" } serde_json = "1" tempfile = "3" diff --git a/tests/suites/conformance/README.md b/tests/suites/conformance/README.md index c51df37c16..a6edb02a7c 100644 --- a/tests/suites/conformance/README.md +++ b/tests/suites/conformance/README.md @@ -9,12 +9,18 @@ nextest run the tests as the library's unit tests. The crate also builds the | Suite | Covers | |---|---| | `smoke` | Create a sandbox, run `echo` in it, delete it | -| `lifecycle` | SDK: long-running and run-to-completion sandboxes, prompt delete when the workload is slow to handle TERM, the default command, restart of a stopped sandbox, deleting a stopped sandbox | +| `lifecycle` | SDK: long-running and run-to-completion sandboxes, prompt delete when the workload is slow to handle TERM, the default command, restart of a stopped sandbox, deleting a stopped sandbox, exec past the startup recovery deadline, the final phase when the main command exits, the main command's environment, `--no-keep` cleanup | | `cli` | The same lifecycle through the `openshell` CLI: status, create, get, list, exec, stop, start, delete | | `persistent-workspace` | Files in `/sandbox` survive stop and start | | `policy-advisor` | `policy.local` accepts a proposed rule; a denied address or hostname yields one draft rule | -| `file-transfer` | `sandbox upload` and `download`: round trips, `.gitignore` filtering, workspace-escape and option-looking names | +| `file-transfer` | `sandbox upload` and `download`: round trips, file modes and symlinks outside the workspace, `.gitignore` filtering, workspace-escape and option-looking names, a large upload | | `credential-gating` | Provider credentials: admission of credentialed endpoints, REST body placeholders and rewriting, WebSocket binary frames, live policy binding | +| `image` | A sandbox runs the default image or the one it is created from | +| `labels` | Labels are stored and select sandboxes in `sandbox list` | +| `templates` | A sandbox created from a template uses its workload; duplicate, deleted and missing templates return the right errors | +| `settings` | Sandbox-scoped settings | +| `process` | Core dumps are disabled for sandbox processes | +| `filesystem-policy` | Landlock: a hard requirement accepts added device paths; a replaced policy path rejects the next launch | `smoke` is the smallest end-to-end check that the runner can run a real test in a tmachine environment on CI. The other suites are ports of the old diff --git a/tests/suites/conformance/src/cli.rs b/tests/suites/conformance/src/cli.rs index d26eed9bb5..f06f16d2d0 100644 --- a/tests/suites/conformance/src/cli.rs +++ b/tests/suites/conformance/src/cli.rs @@ -23,18 +23,9 @@ const MAX_PAGES: usize = 1000; async fn cli_create_exec_list_delete(cx: &mut Context) -> testkit::Result { cx.cli(["status"]).await?.success()?; - let name = cx.name("cs"); - cx.track(&name); - cx.cli(["sandbox", "create", "--name", &name, "--detach"]) - .await? - .success()?; - - let sandbox = cx.adopt(&name).await?; - let got: Value = cx - .cli(["sandbox", "get", &name, "--output", "json"]) - .await? - .success()? - .json()?; + let sandbox = cx.cli_sandbox("cs").create().await?; + let name = sandbox.name().to_string(); + let got = sandbox.get_json().await?; testkit::ensure!( got["name"] == name.as_str(), "sandbox get returned {:?}; expected {name:?}", @@ -50,46 +41,27 @@ async fn cli_create_exec_list_delete(cx: &mut Context) -> testkit::Result { testkit::ensure!(listed.is_some(), "sandbox {name} is not in sandbox list"); let marker = format!("openshell-e2e-{}", cx.run_id()); - let exec = cx - .cli([ - "sandbox", "exec", "--name", &name, "--no-tty", "--", "echo", &marker, - ]) - .await? - .success()?; - testkit::ensure!( - exec.stdout == format!("{marker}\n"), - "sandbox exec stdout was {:?}; expected {:?}", - exec.stdout, - format!("{marker}\n") - ); + cx.cli([ + "sandbox", "exec", "--name", &name, "--no-tty", "--", "echo", &marker, + ]) + .await? + .expect_stdout(&format!("{marker}\n"))?; cx.cli(["sandbox", "delete", &name]).await?.success()?; wait_unlisted(cx, &name).await?; - cx.expect_absent(&sandbox).await + sandbox.expect_absent().await } /// A sandbox created with the CLI can be stopped, rejects `sandbox exec` while /// stopped, starts again, and can be stopped and deleted. #[testkit::test] async fn cli_stop_start_delete(cx: &mut Context) -> testkit::Result { - let name = cx.name("cl"); - cx.track(&name); - cx.cli([ - "sandbox", - "create", - "--name", - &name, - "--detach", - "--no-tty", - "--", - "sh", - "-c", - "exec sleep infinity", - ]) - .await? - .success()?; - let sandbox = cx.adopt(&name).await?; - sandbox.wait_ready().await?; + let sandbox = cx + .cli_sandbox("cl") + .command(["sh", "-c", "exec sleep infinity"]) + .create_ready() + .await?; + let name = sandbox.name().to_string(); cx.cli(["sandbox", "stop", &name]).await?.success()?; sandbox @@ -108,7 +80,7 @@ async fn cli_stop_start_delete(cx: &mut Context) -> testkit::Result { cx.cli(["sandbox", "delete", &name]).await?.success()?; wait_unlisted(cx, &name).await?; - cx.expect_absent(&sandbox).await + sandbox.expect_absent().await } /// Wait until `sandbox list` no longer shows the sandbox. diff --git a/tests/suites/conformance/src/credential_gating.rs b/tests/suites/conformance/src/credential_gating.rs index ea67464e35..2d67926ef0 100644 --- a/tests/suites/conformance/src/credential_gating.rs +++ b/tests/suites/conformance/src/credential_gating.rs @@ -14,7 +14,7 @@ use host_server_support::{HttpRecorder, Request, WebSocketRecorder}; use testkit::policy::{self, Access, Endpoint as _}; use testkit::sdk::raw::proto; use testkit::{ - Context, Error, GrpcCode, PYTHON_BINARIES, Poll, Result, ResultExt as _, Sandbox, YamlFile, + Context, Error, GrpcCode, PYTHON_BINARIES, Result, ResultExt as _, Sandbox, YamlFile, }; const TEST_HOST: &str = "host.openshell.internal"; @@ -298,10 +298,8 @@ fn sandbox_with( ) -> testkit::SandboxBuilder { cx.sandbox(suffix) .policy(policy) - .spec(|spec| { - spec.image = Some(testkit::python_image()); - spec.providers = providers.iter().map(ToString::to_string).collect(); - }) + .image(testkit::python_image()) + .spec(|spec| spec.providers = providers.iter().map(ToString::to_string).collect()) .command(["sleep", "infinity"]) } @@ -313,9 +311,9 @@ async fn create_sandbox( policy: proto::SandboxPolicy, providers: &[&str], ) -> Result { - let sandbox = sandbox_with(cx, suffix, policy, providers).create().await?; - sandbox.wait_ready().await?; - Ok(sandbox) + sandbox_with(cx, suffix, policy, providers) + .create_ready() + .await } /// Fail unless the gateway rejects the sandbox as a failed precondition, and @@ -384,25 +382,15 @@ fn redact(request: &Request) -> String { /// Wait until new exec sessions report `expected` from [`TOKEN_PRESENCE`]. async fn wait_for_token(sandbox: &Sandbox, expected: &str) -> Result { - testkit::poll_until( - &format!("the provider environment to report {expected}"), - Duration::from_secs(30), - Duration::from_millis(500), - async || { - let output = sandbox.exec(["sh", "-lc", TOKEN_PRESENCE]).await?; - Ok(if output.stdout.contains(expected) { - Poll::Ready(()) - } else { - Poll::Pending(format!( - "exit {}; stdout: {}; stderr: {}", - output.exit_code, - output.stdout.trim(), - output.stderr.trim() - )) - }) - }, - ) - .await + sandbox + .poll_exec( + &format!("the provider environment to report {expected}"), + Duration::from_secs(30), + ["sh", "-lc", TOKEN_PRESENCE], + |output| output.stdout.contains(expected), + ) + .await?; + Ok(()) } /// Replace `sandbox`'s policy with `policy` and wait until it is applied. diff --git a/tests/suites/conformance/src/file_transfer.rs b/tests/suites/conformance/src/file_transfer.rs index 5bd24563c0..bfccffff6d 100644 --- a/tests/suites/conformance/src/file_transfer.rs +++ b/tests/suites/conformance/src/file_transfer.rs @@ -11,10 +11,11 @@ //! why they run in every environment. use std::fs; +use std::os::unix::fs::{PermissionsExt, symlink}; use std::path::{Path, PathBuf}; use tempfile::TempDir; -use testkit::{CliOutput, Context, Error, Sandbox}; +use testkit::{CliOutput, Context, Error, ResultExt as _, Sandbox}; const LARGE_FILE_SIZE: usize = 512 * 1024; @@ -25,6 +26,7 @@ const LARGE_FILE_SIZE: usize = 512 * 1024; async fn upload_download_round_trip(cx: &mut Context) -> testkit::Result { let ft = Transfer::new(cx, "fr").await?; ft.upload_into_existing_directories().await?; + ft.upload_metadata_outside_workdir().await?; ft.round_trip().await?; ft.download_file().await?; ft.download_directory().await?; @@ -53,6 +55,84 @@ async fn download_stays_inside_workspace(cx: &mut Context) -> testkit::Result { ft.sandbox.delete().await } +const BACKPRESSURE_FILE_COUNT: usize = 12_000; +const BACKPRESSURE_FILE_BYTES: usize = 8_192; + +/// An upload far larger than the supervisor's 4 MiB pending-stdin budget, +/// made of many small files that keep tar busy creating them, completes on +/// the first attempt with every file intact. The sender is not paced, so the +/// transfer must apply backpressure (#4430). +#[testkit::test] +async fn large_upload_backpressure(cx: &mut Context) -> testkit::Result { + let ft = Transfer::new(cx, "fb").await?; + let source = ft.local("large-project"); + create_dir_all(&source.join("payload"))?; + + // Distinct pseudorandom bytes, so compression or identical-file checks + // can't hide dropped data. + let mut state: u64 = 4430; + let mut payload = vec![0_u8; BACKPRESSURE_FILE_BYTES]; + let mut checksums = String::new(); + for index in 0..BACKPRESSURE_FILE_COUNT { + for chunk in payload.chunks_mut(8) { + chunk.copy_from_slice(&splitmix64(&mut state).to_le_bytes()[..chunk.len()]); + } + let relative = format!("payload/{index:05}.bin"); + fs::write(source.join(&relative), &payload).context(format!("write {relative}"))?; + checksums.push_str(&format!("{} {relative}\n", sha256_hex(&payload)?)); + } + write(&source.join("SHA256SUMS"), &checksums)?; + + let remote = format!("{}/backpressure", ft.remote_root); + let uploaded = ft + .cx + .cli_with_env( + &[("RUST_LOG", "warn")], + [ + "sandbox", + "upload", + ft.sandbox.name(), + &source.display().to_string(), + &remote, + "--no-git-ignore", + ], + ) + .await? + .success()?; + testkit::ensure!( + !uploaded + .report_text() + .contains("sandbox sync operation failed; retrying"), + "the large upload needed a retry: {}", + uploaded.report_text() + ); + + ft.sh(&format!( + "set -eu; cd '{remote}/large-project'; \ + test \"$(find payload -type f | wc -l)\" -eq {BACKPRESSURE_FILE_COUNT}; \ + test \"$(sha256sum SHA256SUMS | cut -d ' ' -f 1)\" = {}; \ + sha256sum --check --quiet SHA256SUMS", + sha256_hex(checksums.as_bytes())? + )) + .await?; + ft.sandbox.delete().await +} + +/// One step of SplitMix64, a small deterministic generator for fixture bytes. +fn splitmix64(state: &mut u64) -> u64 { + *state = state.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) +} + +fn sha256_hex(bytes: &[u8]) -> testkit::Result { + let digest = + openshell_crypto::sha256(bytes).map_err(|error| Error::msg(format!("SHA-256: {error}")))?; + Ok(digest.iter().map(|byte| format!("{byte:02x}")).collect()) +} + /// A sandbox, a directory in its workspace, and a local scratch directory. struct Transfer<'a> { cx: &'a Context, @@ -66,12 +146,10 @@ impl<'a> Transfer<'a> { let sandbox = cx .sandbox(suffix) .command(["sleep", "infinity"]) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; let remote_root = format!("/sandbox/file-transfer-{}-{suffix}", cx.run_id()); - let local = tempfile::tempdir() - .map_err(|error| Error::msg(format!("create a local scratch directory: {error}")))?; + let local = tempfile::tempdir().context("create a local scratch directory")?; let ft = Self { cx, sandbox, @@ -88,14 +166,11 @@ impl<'a> Transfer<'a> { /// Run a shell script in the sandbox and require it to succeed. async fn sh(&self, script: &str) -> testkit::Result { - let output = self.sandbox.exec(["sh", "-c", script]).await?; - testkit::ensure!( - output.exit_code == 0, - "`{script}` exited with {}; stderr: {}", - output.exit_code, - output.stderr - ); - Ok(()) + self.sandbox + .exec(["sh", "-c", script]) + .await? + .success() + .map(drop) } async fn upload( @@ -169,6 +244,51 @@ impl<'a> Transfer<'a> { Ok(()) } + /// An upload to a directory outside the workspace keeps file modes and + /// symlinks, dangling ones included, overwrites files it carries, and + /// leaves other files in the destination alone. + async fn upload_metadata_outside_workdir(&self) -> testkit::Result { + let source = self.local("metadata-project"); + create_dir_all(&source.join("payload"))?; + write(&source.join("payload/00000.bin"), "payload")?; + let script = source.join("run.sh"); + write(&script, "#!/bin/sh\nprintf '%s\\n' upload-executable-ok\n")?; + fs::set_permissions(&script, fs::Permissions::from_mode(0o751)).context("chmod run.sh")?; + for (target, link) in [ + ("payload/00000.bin", "payload-link"), + ("missing-target", "dangling-link"), + ] { + symlink(target, source.join(link)).context(format!("symlink {link}"))?; + } + write(&source.join("conflict.txt"), "local-conflict")?; + + let remote = format!( + "/tmp/file-transfer-{}-{}", + self.cx.run_id(), + self.sandbox.name() + ); + let project = format!("{remote}/metadata-project"); + self.sh(&format!( + "set -eu; test \"$(pwd -P)\" != '{remote}'; mkdir -p '{project}'; \ + printf remote-conflict > '{project}/conflict.txt'; \ + printf remote-preserved > '{project}/unrelated.txt'" + )) + .await?; + + self.upload(&source, &remote, true).await?.success()?; + self.sh(&format!( + "set -eu; cd '{project}'; \ + test \"$(stat -c %a run.sh)\" = 751; \ + test \"$(./run.sh)\" = upload-executable-ok; \ + test -L payload-link; test \"$(readlink payload-link)\" = payload/00000.bin; \ + test -L dangling-link; test ! -e dangling-link; \ + test \"$(readlink dangling-link)\" = missing-target; \ + test \"$(cat conflict.txt)\" = local-conflict; \ + test \"$(cat unrelated.txt)\" = remote-preserved" + )) + .await + } + /// A directory with a nested file and a large binary file, and a single /// file, survive upload then download. async fn round_trip(&self) -> testkit::Result { @@ -177,12 +297,11 @@ impl<'a> Transfer<'a> { write(&source.join("greeting.txt"), "hello-from-local")?; write(&source.join("subdir/nested.txt"), "nested-content")?; let large: Vec = (0u8..=250).cycle().take(LARGE_FILE_SIZE).collect(); - fs::write(source.join("large.bin"), &large) - .map_err(|error| Error::msg(format!("write large.bin: {error}")))?; + fs::write(source.join("large.bin"), &large).context("write large.bin")?; let remote = format!("{}/roundtrip", self.remote_root); let uploaded = self.upload(&source, &remote, false).await?.success()?; - let warning = message(&uploaded); + let warning = uploaded.report_text(); testkit::ensure!( warning.contains("outside a Git work tree") && warning.contains(".gitignore rules are not applied"), @@ -196,8 +315,7 @@ impl<'a> Transfer<'a> { .success()?; require_text(&destination.join("greeting.txt"), "hello-from-local")?; require_text(&destination.join("subdir/nested.txt"), "nested-content")?; - let actual = fs::read(destination.join("large.bin")) - .map_err(|error| Error::msg(format!("read large.bin: {error}")))?; + let actual = fs::read(destination.join("large.bin")).context("read large.bin")?; testkit::ensure!( actual == large, "large file changed in the round trip: sent {} bytes, received {}", @@ -310,7 +428,7 @@ impl<'a> Transfer<'a> { let remote = format!("{}/reuploaded", self.remote_root); let rejected = self.upload(&runs, &remote, false).await?.failure()?; - let output = message(&rejected); + let output = rejected.report_text(); testkit::ensure!( output.contains("filtering selected no files") && output.contains("--no-git-ignore"), "the rejected upload did not explain the --no-git-ignore override: {output}" @@ -336,7 +454,7 @@ impl<'a> Transfer<'a> { for source in [etc_link.clone(), passwd_link, format!("{etc_link}/passwd")] { let rejected = self.download(&source, &destination).await?.failure()?; - let output = message(&rejected); + let output = rejected.report_text(); testkit::ensure!( output.contains("resolves to") && output.contains("outside the") @@ -362,22 +480,12 @@ impl<'a> Transfer<'a> { } } -/// The CLI's stdout and stderr as one line of words. Error reports wrap to -/// the terminal width and draw a gutter, so match against this instead. -fn message(output: &CliOutput) -> String { - format!("{} {}", output.stdout, output.stderr) - .replace(['│', '×', '╰', '─', '╭', '├'], " ") - .split_whitespace() - .collect::>() - .join(" ") -} - fn git(repository: &Path, args: &[&str]) -> testkit::Result { let output = std::process::Command::new("git") .args(args) .current_dir(repository) .output() - .map_err(|error| Error::msg(format!("run git {}: {error}", args.join(" "))))?; + .context(format!("run git {}", args.join(" ")))?; testkit::ensure!( output.status.success(), "git {} failed in {}: {}", @@ -389,18 +497,15 @@ fn git(repository: &Path, args: &[&str]) -> testkit::Result { } fn write(path: &Path, contents: &str) -> testkit::Result { - fs::write(path, contents) - .map_err(|error| Error::msg(format!("write {}: {error}", path.display()))) + fs::write(path, contents).context(format!("write {}", path.display())) } fn create_dir_all(path: &Path) -> testkit::Result { - fs::create_dir_all(path) - .map_err(|error| Error::msg(format!("create {}: {error}", path.display()))) + fs::create_dir_all(path).context(format!("create {}", path.display())) } fn require_text(path: &Path, expected: &str) -> testkit::Result { - let actual = fs::read_to_string(path) - .map_err(|error| Error::msg(format!("read {}: {error}", path.display())))?; + let actual = fs::read_to_string(path).context(format!("read {}", path.display()))?; testkit::ensure!( actual == expected, "{} contains {actual:?}; expected {expected:?}", diff --git a/tests/suites/conformance/src/filesystem_policy.rs b/tests/suites/conformance/src/filesystem_policy.rs new file mode 100644 index 0000000000..79e6937ad8 --- /dev/null +++ b/tests/suites/conformance/src/filesystem_policy.rs @@ -0,0 +1,156 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Filesystem policy suite: Landlock enforcement of a sandbox's filesystem +//! policy. + +use std::collections::HashMap; +use std::time::Duration; + +use testkit::sdk::raw::proto; +use testkit::{Context, ResultExt as _, with_deadline}; + +/// A rejected launch must close promptly instead of hanging the exec. +const REJECTED_LAUNCH_TIMEOUT: Duration = Duration::from_secs(20); + +/// The filesystem policy the tests start from, with `extra_read_write` added +/// to the writable paths. It names no device paths; the sandbox adds the ones +/// every process needs, such as `/dev/urandom`. +/// +/// It sets no process identity, so the workload runs as the image's `USER`: +/// Landlock applies whatever the identity, and the test image need not have a +/// particular user. +fn policy(compatibility: &str, extra_read_write: &[&str]) -> proto::SandboxPolicy { + let paths = |paths: &[&str]| paths.iter().map(ToString::to_string).collect::>(); + let mut read_write = paths(&["/sandbox"]); + read_write.extend(paths(extra_read_write)); + read_write.push("/tmp".to_string()); + proto::SandboxPolicy { + version: 1, + filesystem: Some(proto::FilesystemPolicy { + include_workdir: true, + read_only: paths(&["/usr", "/lib", "/etc", "/proc"]), + read_write, + }), + landlock: Some(proto::LandlockPolicy { + compatibility: compatibility.to_string(), + }), + network_policies: HashMap::from([( + "landlock_regression".to_string(), + proto::NetworkPolicyRule { + name: "landlock_regression".to_string(), + endpoints: vec![proto::NetworkEndpoint { + host: "example.com".to_string(), + port: 443, + ..proto::NetworkEndpoint::default() + }], + binaries: vec![proto::NetworkBinary { + path: "/**".to_string(), + }], + }, + )]), + ..proto::SandboxPolicy::default() + } +} + +/// Under `hard_requirement`, the device paths the sandbox adds to the policy +/// don't make Landlock setup fail: processes start, read `/dev/urandom`, and +/// write to a writable path. +#[testkit::test] +async fn hard_requirement_accepts_added_device_paths(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("ld") + .command(["sleep", "infinity"]) + .policy(policy("hard_requirement", &[])) + .create_ready() + .await?; + + sandbox + .exec([ + "sh", + "-c", + "set -eu; \ + bytes=$(head -c 16 /dev/urandom | wc -c); \ + test \"$bytes\" -eq 16; \ + printf landlock-ok > /tmp/landlock-check; \ + test \"$(cat /tmp/landlock-check)\" = landlock-ok", + ]) + .await? + .success() + .context("the hard_requirement sandbox must read /dev/urandom and write /tmp")?; + Ok(()) +} + +/// When a workload replaces a policy path, or one of its ancestors, with a +/// symlink, the next launch is rejected rather than following the symlink, +/// in both Landlock compatibility modes. +#[testkit::test] +async fn replaced_policy_path_rejects_next_launch(cx: &mut Context) -> testkit::Result { + const LEAF: &str = "/sandbox/policy-parent/leaf"; + let cases = [ + ("lbp", "best_effort", false), + ("lhp", "hard_requirement", false), + ("lba", "best_effort", true), + ("lha", "hard_requirement", true), + ]; + + let mut sandboxes = Vec::new(); + for (suffix, compatibility, ancestor) in cases { + let sandbox = cx + .sandbox(suffix) + .command(["sleep", "infinity"]) + .policy(policy(compatibility, &[LEAF])) + .create() + .await?; + sandboxes.push((sandbox, compatibility, ancestor)); + } + + for (sandbox, compatibility, ancestor) in &sandboxes { + let case = format!( + "{compatibility}, replacing the {}", + if *ancestor { "ancestor" } else { "path" } + ); + sandbox.wait_ready().await?; + for script in [ + format!("mkdir -p {LEAF}"), + // A normal directory stays usable on the next launch. + format!("echo allowed > {LEAF}/check"), + if *ancestor { + "mv /sandbox/policy-parent /sandbox/original-parent; \ + ln -s /var/tmp /sandbox/policy-parent" + .to_string() + } else { + format!("mv {LEAF} /sandbox/original-leaf; ln -s /var/tmp {LEAF}") + }, + ] { + sandbox + .exec(["sh", "-c", script.as_str()]) + .await? + .success() + .context(format!("({case}) prepare the policy path"))?; + } + + // The exec's own error is the rejection report, so keep it apart from + // the deadline's. + let next = with_deadline( + &format!("({case}) the rejected launch"), + REJECTED_LAUNCH_TIMEOUT, + async { Ok(sandbox.exec(["true"]).await) }, + ) + .await?; + let report = match next { + Ok(output) if output.exit_code == 0 => { + return Err(testkit::Error::msg(format!( + "({case}) a launch succeeded after a policy path was replaced with a symlink" + ))); + } + Ok(output) => output.stderr, + Err(error) => error.to_string(), + }; + testkit::ensure!( + report.to_lowercase().contains("symlink"), + "({case}) the rejected launch does not say a policy path is a symlink: {report}" + ); + } + Ok(()) +} diff --git a/tests/suites/conformance/src/image.rs b/tests/suites/conformance/src/image.rs new file mode 100644 index 0000000000..37286e5bfc --- /dev/null +++ b/tests/suites/conformance/src/image.rs @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox image suite: a sandbox runs the image it is created from. + +use testkit::{Context, ResultExt as _, Sandbox}; + +/// A sandbox created with an explicit image runs that image. +#[testkit::test] +async fn explicit_image_is_used(cx: &mut Context) -> testkit::Result { + let image = testkit::base_image(); + let sandbox = cx + .sandbox("ei") + .command(["sleep", "infinity"]) + .image(image.clone()) + .create_ready() + .await?; + expect_ubuntu(&sandbox) + .await + .context(format!("sandbox from image {image}")) +} + +/// Fail unless the sandbox's `/etc/os-release` names Ubuntu and a version, +/// as OpenShell's base image does. +pub(crate) async fn expect_ubuntu(sandbox: &Sandbox) -> testkit::Result { + let output = sandbox.exec(["cat", "/etc/os-release"]).await?.success()?; + let field = |key: &str| { + output.stdout.lines().find_map(|line| { + line.strip_prefix(key) + .and_then(|rest| rest.strip_prefix('=')) + .map(|value| value.trim_matches('"')) + }) + }; + testkit::ensure!( + field("ID") == Some("ubuntu"), + "/etc/os-release has ID {:?}; expected \"ubuntu\":\n{}", + field("ID"), + output.stdout + ); + testkit::ensure!( + field("VERSION_ID").is_some_and(|version| !version.is_empty()), + "/etc/os-release has no VERSION_ID:\n{}", + output.stdout + ); + Ok(()) +} diff --git a/tests/suites/conformance/src/labels.rs b/tests/suites/conformance/src/labels.rs new file mode 100644 index 0000000000..f60e7f68ea --- /dev/null +++ b/tests/suites/conformance/src/labels.rs @@ -0,0 +1,92 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox labels suite: labels are stored and select sandboxes in list. + +use std::collections::{BTreeSet, HashMap}; + +use testkit::Context; +use testkit::sdk::ListOptions; + +/// Labels given at create are returned by get, and list filters by one label +/// or by several (all must match). Label values are unique to the run, so +/// other tests' sandboxes can't match. +#[testkit::test] +async fn labels_are_stored_and_filter_list(cx: &mut Context) -> testkit::Result { + let run = cx.run_id().to_string(); + let dev = format!("dev-{run}"); + let staging = format!("staging-{run}"); + let backend = format!("backend-{run}"); + let data = format!("data-{run}"); + let labeled = async |suffix: &str, env: &str, team: &str| -> testkit::Result { + let labels = HashMap::from([ + ("env".to_string(), env.to_string()), + ("team".to_string(), team.to_string()), + ]); + let sandbox = cx + .sandbox(suffix) + .spec(|spec| spec.labels = labels.clone()) + .create_ready() + .await?; + let got = sandbox.get().await?; + testkit::ensure!( + labels + .iter() + .all(|(key, value)| got.labels.get(key) == Some(value)), + "sandbox {} has labels {:?}; expected {labels:?}", + sandbox.name(), + got.labels + ); + Ok(sandbox.name().to_string()) + }; + let dev_backend = labeled("ldb", &dev, &backend).await?; + let staging_backend = labeled("lsb", &staging, &backend).await?; + let dev_data = labeled("ldd", &dev, &data).await?; + + let checks = [ + (format!("env={dev}"), vec![&dev_backend, &dev_data]), + ( + format!("team={backend}"), + vec![&dev_backend, &staging_backend], + ), + (format!("env={dev},team={backend}"), vec![&dev_backend]), + (format!("team=qa-{run}"), vec![]), + ]; + for (selector, expected) in checks { + let listed = listed_names( + cx, + ListOptions { + label_selector: Some(selector.clone()), + ..ListOptions::default() + }, + ) + .await?; + let expected: BTreeSet = expected.into_iter().cloned().collect(); + testkit::ensure!( + listed == expected, + "list with selector {selector:?} returned {listed:?}; expected {expected:?}" + ); + } + + let all = listed_names(cx, ListOptions::default()).await?; + let missing: Vec<&String> = [&dev_backend, &staging_backend, &dev_data] + .into_iter() + .filter(|name| !all.contains(*name)) + .collect(); + testkit::ensure!( + missing.is_empty(), + "list without a selector is missing {missing:?}" + ); + Ok(()) +} + +/// The names of the sandboxes `list` returns, across all pages. +async fn listed_names(cx: &Context, options: ListOptions) -> testkit::Result> { + Ok(cx + .client() + .list_all_sandboxes(options) + .await? + .into_iter() + .map(|sandbox| sandbox.name) + .collect()) +} diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index 98b916092f..cabd7f3a3b 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -11,7 +11,13 @@ mod cli; mod credential_gating; mod file_transfer; +mod filesystem_policy; +mod image; +mod labels; mod lifecycle; mod persistent_workspace; mod policy_advisor; +mod process; +mod settings; mod smoke; +mod templates; diff --git a/tests/suites/conformance/src/lifecycle.rs b/tests/suites/conformance/src/lifecycle.rs index 6af1d145b3..71854d2b52 100644 --- a/tests/suites/conformance/src/lifecycle.rs +++ b/tests/suites/conformance/src/lifecycle.rs @@ -9,8 +9,8 @@ use std::time::Duration; -use testkit::Context; use testkit::sdk::{SandboxPhase, SandboxRestartPolicy}; +use testkit::{Context, ResultExt as _, Sandbox, with_deadline}; /// A long-running sandbox becomes ready, runs a command with exec, and is gone /// from get and list after delete. @@ -19,27 +19,54 @@ async fn long_running_exec_delete(cx: &mut Context) -> testkit::Result { let sandbox = cx .sandbox("lr") .command(["sleep", "infinity"]) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; let marker = format!("openshell-e2e-{}", cx.run_id()); - let output = sandbox.exec(["echo", marker.as_str()]).await?; - testkit::ensure!( - output.exit_code == 0, - "exec exited with {}; stderr: {}", - output.exit_code, - output.stderr - ); - testkit::ensure!( - output.stdout == format!("{marker}\n"), - "exec stdout was {:?}; expected {:?}", - output.stdout, - format!("{marker}\n") - ); + sandbox + .exec(["echo", marker.as_str()]) + .await? + .expect_stdout(&format!("{marker}\n"))?; sandbox.delete().await?; - cx.expect_absent(&sandbox).await + sandbox.expect_absent().await +} + +/// Longer than the supervisor's 30s startup recovery deadline. +const PAST_RECOVERY_DEADLINE: Duration = Duration::from_secs(31); +/// The exec's own timeout is the runner's two-minute kill, so fail sooner, +/// with a message that says which step hung. +const PAST_RECOVERY_EXEC_DEADLINE: Duration = Duration::from_mins(1); + +/// An exec admitted before the startup recovery deadline expires keeps +/// running past it, and a new exec still works afterwards. +#[testkit::test] +async fn exec_outlives_startup_recovery_deadline(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("rd") + .command(["sleep", "infinity"]) + .create_ready() + .await?; + + let script = format!( + "sleep {}; printf past-recovery-deadline", + PAST_RECOVERY_DEADLINE.as_secs() + ); + with_deadline( + "the exec past the recovery deadline", + PAST_RECOVERY_EXEC_DEADLINE, + sandbox.exec(["sh", "-c", script.as_str()]), + ) + .await? + .expect_stdout("past-recovery-deadline") + .context("the recovery deadline must not end an admitted exec")?; + + sandbox + .exec(["printf", "fresh-exec"]) + .await? + .expect_stdout("fresh-exec") + .context("exec after the recovery deadline")?; + Ok(()) } /// Long enough that the workload is still handling TERM when sandbox shutdown @@ -67,45 +94,31 @@ async fn delete_is_prompt_when_workload_is_slow_to_handle_term( let sandbox = cx .sandbox("dt") .command(["sh", "-c", script.as_str()]) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; // Ready alone does not prove that the workload has installed its trap. - testkit::poll_until( - "workload TERM handler readiness", - Duration::from_secs(10), - Duration::from_millis(100), - async || { - let output = sandbox - .exec(["sh", "-c", "test -f /tmp/delayed-term-ready"]) - .await?; - Ok(if output.exit_code == 0 { - testkit::Poll::Ready(()) - } else { - testkit::Poll::Pending(format!( - "readiness check exited {}; stderr: {}", - output.exit_code, output.stderr - )) - }) - }, - ) - .await?; + sandbox + .poll_exec( + "workload TERM handler readiness", + Duration::from_secs(10), + ["test", "-f", "/tmp/delayed-term-ready"], + |output| output.exit_code == 0, + ) + .await?; // Use one deadline starting before delete, not a fresh timeout per step. // Boundary containment may kill the workload before its handler finishes; // this asserts bounded deletion, not a graceful workload exit. - tokio::time::timeout(DELETE_DEADLINE, async { - sandbox.delete().await?; - cx.expect_absent(&sandbox).await - }) + with_deadline( + &format!("deleting sandbox {}", sandbox.name()), + DELETE_DEADLINE, + async { + sandbox.delete().await?; + sandbox.expect_absent().await + }, + ) .await - .map_err(|_| { - testkit::Error::msg(format!( - "sandbox {} was not fully deleted within {DELETE_DEADLINE:?}", - sandbox.name() - )) - })? } /// A sandbox whose main command exits 0, with restart policy Never, reaches @@ -148,7 +161,68 @@ async fn run_to_completion_exit_delete(cx: &mut Context) -> testkit::Result { ); sandbox.delete().await?; - cx.expect_absent(&sandbox).await + sandbox.expect_absent().await +} + +/// A main command that exits during provisioning is reported as the +/// workload's result, not as a provisioning failure: the create succeeds and +/// the sandbox reaches Error with the command's exit code. +#[testkit::test] +async fn main_exit_during_provisioning_is_workload_result(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("pe") + .command(["sh", "-c", "exit 11"]) + .restart_policy(SandboxRestartPolicy::Never) + .create() + .await?; + + let finished = sandbox.wait_finished().await?; + testkit::ensure!( + finished.phase == SandboxPhase::Error && finished.exit_code == Some(11), + "sandbox finished in phase {:?} with exit code {:?}; expected Error with Some(11)", + finished.phase, + finished.exit_code + ); + Ok(()) +} + +/// A main command that exits after the sandbox is Ready leaves its result: +/// exit 0 reaches Completed, and a non-zero exit reaches Error with that code. +#[testkit::test] +async fn main_exit_after_ready_sets_final_phase(cx: &mut Context) -> testkit::Result { + const RELEASE_PATH: &str = "/sandbox/.openshell-main-release"; + let cases = [ + ("mz", 0, SandboxPhase::Completed), + ("mn", 11, SandboxPhase::Error), + ]; + + // Create every sandbox before waiting, so they provision concurrently. + let mut sandboxes = Vec::new(); + for (suffix, code, phase) in cases { + let script = + format!("while [ ! -e '{RELEASE_PATH}' ]; do sleep 0.05; done; sleep 1; exit {code}"); + let sandbox = cx + .sandbox(suffix) + .command(["sh", "-c", script.as_str()]) + .restart_policy(SandboxRestartPolicy::Never) + .create() + .await?; + sandboxes.push((sandbox, code, phase)); + } + + for (sandbox, code, phase) in &sandboxes { + sandbox.wait_ready().await?; + sandbox.exec(["touch", RELEASE_PATH]).await?.success()?; + let finished = sandbox.wait_finished().await?; + testkit::ensure!( + finished.phase == *phase && finished.exit_code == Some(*code), + "main exiting {code} left sandbox {} in phase {:?} with exit code {:?}; expected {phase:?} with Some({code})", + sandbox.name(), + finished.phase, + finished.exit_code + ); + } + Ok(()) } /// A sandbox created with the gateway's default command becomes Ready, is @@ -163,20 +237,18 @@ async fn default_command_sandbox_is_listed_ready(cx: &mut Context) -> testkit::R ready.name, sandbox.name() ); - cx.expect_listed(&sandbox, SandboxPhase::Ready).await?; + sandbox.expect_listed(SandboxPhase::Ready).await?; let marker = format!("openshell-e2e-{}", cx.run_id()); - let output = sandbox.exec(["echo", marker.as_str()]).await?; - testkit::ensure!( - output.exit_code == 0 && output.stdout == format!("{marker}\n"), - "exec exited {} with stdout {:?}; stderr: {}", - output.exit_code, - output.stdout, - output.stderr - ); + sandbox + .exec(["echo", marker.as_str()]) + .await? + .expect_stdout(&format!("{marker}\n"))?; + // The gateway's default image is OpenShell's Ubuntu base image. + crate::image::expect_ubuntu(&sandbox).await?; sandbox.delete().await?; - cx.expect_absent(&sandbox).await + sandbox.expect_absent().await } /// A stopped sandbox rejects exec, accepts a second stop, and returns to @@ -186,18 +258,15 @@ async fn restart_stopped_sandbox(cx: &mut Context) -> testkit::Result { let sandbox = cx .sandbox("rs") .command(["sleep", "infinity"]) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; sandbox.stop().await?; - match sandbox.exec(["true"]).await { - Err(_) => {} - Ok(output) => testkit::ensure!( - output.exit_code != 0, - "exec succeeded while the sandbox was stopped" - ), - } + let rejected = sandbox.exec(["true"]).await; + testkit::ensure!( + !matches!(&rejected, Ok(output) if output.exit_code == 0), + "exec succeeded while the sandbox was stopped" + ); // Stop is idempotent. let again = sandbox.stop().await?; testkit::ensure!( @@ -207,14 +276,11 @@ async fn restart_stopped_sandbox(cx: &mut Context) -> testkit::Result { ); sandbox.start().await?; - let output = sandbox.exec(["echo", "started"]).await?; - testkit::ensure!( - output.exit_code == 0 && output.stdout == "started\n", - "exec after start exited {} with stdout {:?}; stderr: {}", - output.exit_code, - output.stdout, - output.stderr - ); + sandbox + .exec(["echo", "started"]) + .await? + .expect_stdout("started\n") + .context("exec after start")?; Ok(()) } @@ -224,10 +290,142 @@ async fn stopped_sandbox_can_be_deleted(cx: &mut Context) -> testkit::Result { let sandbox = cx .sandbox("sd") .command(["sleep", "infinity"]) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; sandbox.stop().await?; sandbox.delete().await?; - cx.expect_absent(&sandbox).await + sandbox.expect_absent().await +} + +/// Where the main command of [`canonical_main_environment`] records what it +/// sees. +const MAIN_ENV_PATH: &str = "/sandbox/.openshell-main-env"; + +/// Declared environment reaches both the main command and exec, with and +/// without a TTY. A TTY main command also gets the sandbox user's `HOME` and +/// `USER`, not the supervisor's, and a usable `TERM`. +#[testkit::test] +async fn canonical_main_environment(cx: &mut Context) -> testkit::Result { + let declared = format!("present-{}", cx.run_id()); + let script = format!( + "printf 'declared=%s\\nhome=%s\\nuser=%s\\nterm=%s\\n' \ + \"${{REPRO_SENTINEL:-missing}}\" \"$HOME\" \"$USER\" \"$TERM\" > '{MAIN_ENV_PATH}'; \ + exec sleep infinity" + ); + + let mut sandboxes = Vec::new(); + for (suffix, tty) in [("et", true), ("en", false)] { + let sandbox = cx + .sandbox(suffix) + .command(["sh", "-c", script.as_str()]) + .spec(|spec| { + spec.tty = tty; + spec.environment + .insert("REPRO_SENTINEL".to_string(), declared.clone()); + }) + .create() + .await?; + sandboxes.push((sandbox, tty)); + } + + for (sandbox, tty) in &sandboxes { + sandbox.wait_ready().await?; + let main = main_environment(sandbox).await?; + let field = |name: &str| main.get(name).map_or("", String::as_str); + testkit::ensure!( + field("declared") == declared, + "the main command (tty {tty}) saw REPRO_SENTINEL={:?}; expected {declared:?}", + field("declared") + ); + + sandbox + .exec(["sh", "-c", "printf '%s' \"${REPRO_SENTINEL:-missing}\""]) + .await? + .expect_stdout(&declared) + .context(format!("REPRO_SENTINEL in exec (tty {tty})"))?; + + if *tty { + testkit::ensure!( + !field("home").is_empty() && field("home") != "/root", + "the TTY main command inherited the supervisor's HOME: {main:?}" + ); + testkit::ensure!( + !field("user").is_empty(), + "the TTY main command has no USER: {main:?}" + ); + testkit::ensure!( + !field("term").is_empty() && field("term") != "dumb", + "the TTY main command has no usable TERM: {main:?}" + ); + } + } + Ok(()) +} + +/// The `key=value` lines the main command wrote to [`MAIN_ENV_PATH`]. The +/// sandbox can be Ready before the main command writes them. +async fn main_environment( + sandbox: &Sandbox, +) -> testkit::Result> { + let output = sandbox + .poll_exec( + &format!( + "the main command of {} to record its environment", + sandbox.name() + ), + Duration::from_secs(30), + ["cat", MAIN_ENV_PATH], + |output| output.exit_code == 0 && output.stdout.contains("term="), + ) + .await?; + Ok(output + .stdout + .lines() + .filter_map(|line| line.split_once('=')) + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect()) +} + +/// How long the gateway may take to remove an ephemeral sandbox after its +/// main command exits. Both sandboxes are released before the test waits, so +/// their removals overlap. +const EPHEMERAL_CLEANUP_TIMEOUT: Duration = Duration::from_secs(60); + +/// A detached sandbox created with `--no-keep` is removed once its main +/// command exits, whether the command succeeds or fails. +/// +/// This checks only what the gateway shows: get and list stop returning the +/// sandbox. Conformance can't see driver resources, so the check that the +/// driver removed its containers, pods or VM state stays in the legacy +/// `e2e/rust/tests/ephemeral_cleanup.rs` until the driver suites replace it. +#[testkit::test] +async fn detached_no_keep_sandbox_is_removed_after_main_exits(cx: &mut Context) -> testkit::Result { + let mut sandboxes = Vec::new(); + for (suffix, code) in [("ez", 0), ("ef", 17)] { + let builder = cx.cli_sandbox(suffix); + let release_path = format!("/sandbox/.ephemeral-release-{}", builder.name()); + let script = + format!("while [ ! -e '{release_path}' ]; do sleep 0.1; done; sleep 1; exit {code}"); + let sandbox = builder + .args(["--no-keep"]) + .command(["sh", "-c", &script]) + .create() + .await?; + sandboxes.push((sandbox, release_path, code)); + } + + for (sandbox, release_path, _) in &sandboxes { + sandbox.wait_ready().await?; + sandbox + .exec(["touch", release_path.as_str()]) + .await? + .success()?; + } + for (sandbox, _, code) in &sandboxes { + sandbox + .wait_absent(EPHEMERAL_CLEANUP_TIMEOUT) + .await + .context(format!("ephemeral sandbox exited {code}"))?; + } + Ok(()) } diff --git a/tests/suites/conformance/src/persistent_workspace.rs b/tests/suites/conformance/src/persistent_workspace.rs index de2940e571..1b123fe618 100644 --- a/tests/suites/conformance/src/persistent_workspace.rs +++ b/tests/suites/conformance/src/persistent_workspace.rs @@ -10,7 +10,7 @@ use std::time::Duration; -use testkit::{Context, Poll}; +use testkit::{Context, ResultExt as _}; /// Files in the workspace survive stop and start, and start runs the main /// command again. @@ -26,50 +26,37 @@ async fn persistent_workspace_survives_stop_start(cx: &mut Context) -> testkit:: let sandbox = cx .sandbox("kw") .command(["sh", "-c", main.as_str()]) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; let sentinel = format!("openshell-stop-start-{}", cx.run_id()); - let write = sandbox + sandbox .exec([ "sh", "-c", &format!("printf '%s\\n' '{sentinel}' > '{SENTINEL}' && sync"), ]) - .await?; - testkit::ensure!( - write.exit_code == 0, - "writing the sentinel exited {}; stderr: {}", - write.exit_code, - write.stderr - ); + .await? + .success() + .context("write the sentinel")?; sandbox.stop().await?; sandbox.start().await?; - let read = sandbox.exec(["cat", SENTINEL]).await?; - testkit::ensure!( - read.stdout == format!("{sentinel}\n"), - "sentinel after restart is {:?}; stderr: {}", - read.stdout, - read.stderr - ); + sandbox + .exec(["cat", SENTINEL]) + .await? + .expect_stdout(&format!("{sentinel}\n")) + .context("read the sentinel after restart")?; // The sandbox is Ready before the restarted main command has necessarily // rewritten the count, so wait for it to say it ran again. - testkit::poll_until( - "the main command to run again", - Duration::from_secs(30), - Duration::from_secs(1), - async || { - let runs = sandbox.exec(["cat", RUN_COUNT]).await?; - Ok(if runs.stdout == "2\n" { - Poll::Ready(()) - } else { - Poll::Pending(format!("run count {:?}", runs.stdout.trim())) - }) - }, - ) - .await?; + sandbox + .poll_exec( + "the main command to run again", + Duration::from_secs(30), + ["cat", RUN_COUNT], + |runs| runs.stdout == "2\n", + ) + .await?; Ok(()) } diff --git a/tests/suites/conformance/src/policy_advisor.rs b/tests/suites/conformance/src/policy_advisor.rs index 003917163c..82ff54cc09 100644 --- a/tests/suites/conformance/src/policy_advisor.rs +++ b/tests/suites/conformance/src/policy_advisor.rs @@ -13,7 +13,7 @@ use std::time::{Duration, Instant}; use serde_json::Value; use testkit::sdk::raw::proto; -use testkit::{Context, Error, Sandbox}; +use testkit::{Context, Error, ResultExt as _, Sandbox}; const READY_TIMEOUT: Duration = Duration::from_mins(4); const PROPOSAL_TIMEOUT: Duration = Duration::from_secs(90); @@ -161,12 +161,13 @@ async fn denied_address_produces_draft_rule(cx: &mut Context) -> testkit::Result "printf 'BINARY=%s\\n' \"$(readlink -f /proc/$$/exe)\"; \ if exec 3<>/dev/tcp/1.1.1.1/443; then echo UNEXPECTED_ALLOWED; exit 1; else echo DENIED; fi", ]) - .await?; + .await? + .success() + .context("the connection to 1.1.1.1:443 must be denied")?; testkit::ensure!( - probe.exit_code == 0 && probe.stdout.lines().any(|line| line == "DENIED"), - "the connection to 1.1.1.1:443 was not denied: stdout {:?}, stderr {:?}", - probe.stdout, - probe.stderr + probe.stdout.lines().any(|line| line == "DENIED"), + "the connection to 1.1.1.1:443 was not denied: stdout {:?}", + probe.stdout ); let binary = probe .stdout @@ -199,12 +200,13 @@ async fn denied_hostname_produces_draft_rule(cx: &mut Context) -> testkit::Resul "-c", "if exec 3<>/dev/tcp/pypi.org/80; then echo UNEXPECTED_ALLOWED; exit 1; else echo DENIED; fi", ]) - .await?; + .await? + .success() + .context("the connection to pypi.org:80 must be denied")?; testkit::ensure!( - probe.exit_code == 0 && probe.stdout.lines().any(|line| line == "DENIED"), - "the connection to pypi.org:80 was not denied: stdout {:?}, stderr {:?}", - probe.stdout, - probe.stderr + probe.stdout.lines().any(|line| line == "DENIED"), + "the connection to pypi.org:80 was not denied: stdout {:?}", + probe.stdout ); expect_draft( @@ -228,8 +230,7 @@ async fn create( if let Some(policy) = policy { builder = builder.policy(policy); } - let sandbox = builder.create().await?; - sandbox.wait_ready().await?; + let sandbox = builder.create_ready().await?; Ok(sandbox) } @@ -289,13 +290,13 @@ async fn bash_path(sandbox: &Sandbox) -> testkit::Result { "-c", "printf '%s\\n' \"$(readlink -f /proc/$$/exe)\"", ]) - .await?; + .await? + .success()?; let path = output.stdout.trim(); testkit::ensure!( - output.exit_code == 0 && is_bash_path(path), - "expected one absolute Bash path; got {:?}, stderr {:?}", - output.stdout, - output.stderr + is_bash_path(path), + "expected one absolute Bash path; got {:?}", + output.stdout ); Ok(path.to_string()) } @@ -326,13 +327,9 @@ async fn policy_local( path, body, ]) - .await?; - testkit::ensure!( - output.exit_code == 0, - "{method} http://policy.local{path} exited {}; stderr: {}", - output.exit_code, - output.stderr - ); + .await? + .success() + .context(format!("{method} http://policy.local{path}"))?; let (headers, json) = output.stdout.split_once("\r\n\r\n").ok_or_else(|| { Error::msg(format!( "{method} http://policy.local{path} returned no complete HTTP response: {:?}", diff --git a/tests/suites/conformance/src/process.rs b/tests/suites/conformance/src/process.rs new file mode 100644 index 0000000000..1f7e9d1eae --- /dev/null +++ b/tests/suites/conformance/src/process.rs @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox process suite: limits the sandbox applies to its processes. + +use testkit::{Context, ResultExt as _}; + +/// Sandbox processes can't write core dumps: both the soft and the hard core +/// file size limits are 0, so a process can't raise its own limit either. +#[testkit::test] +async fn core_dumps_are_disabled(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("cd") + .command(["sleep", "infinity"]) + .create_ready() + .await?; + + sandbox + .exec([ + "sh", + "-c", + "printf '%s %s' \"$(ulimit -c)\" \"$(ulimit -Hc)\"", + ]) + .await? + .expect_stdout("0 0") + .context("core file size limits (soft hard)")?; + Ok(()) +} diff --git a/tests/suites/conformance/src/settings.rs b/tests/suites/conformance/src/settings.rs new file mode 100644 index 0000000000..632356fc09 --- /dev/null +++ b/tests/suites/conformance/src/settings.rs @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Settings suite: sandbox-scoped settings. +//! +//! Conformance tests only read gateway-global settings. The global override +//! test changes a setting for every sandbox on the gateway, so it stays in +//! `e2e/rust/tests/settings_management.rs` until #5527 decides where +//! destructive tests run. + +use std::time::Duration; + +use serde_json::Value; +use testkit::{Context, Poll, Sandbox, poll_until}; + +/// A registered boolean setting that does not change what these tests do. +const KEY: &str = "ocsf_json_enabled"; +/// How the CLI shows a setting with no value. +const UNSET: &str = ""; + +/// With no global value, a sandbox setting starts unset, reads back after it +/// is set, and is unset again after it is deleted. +#[testkit::test] +async fn sandbox_setting_round_trip(cx: &mut Context) -> testkit::Result { + // A global value would hide the sandbox's own. + let global = read_global_setting(cx, KEY).await?; + testkit::ensure!( + global == UNSET, + "the gateway sets {KEY} globally ({global}), which hides sandbox values; delete it with `openshell settings delete --global --key {KEY}`" + ); + let sandbox = cx + .sandbox("sr") + .command(["sleep", "infinity"]) + .create_ready() + .await?; + expect_sandbox_setting(cx, &sandbox, UNSET, "unset").await?; + + set_sandbox_setting(cx, &sandbox, "true").await?.success()?; + expect_sandbox_setting(cx, &sandbox, "true", "sandbox").await?; + + cx.cli(["settings", "delete", sandbox.name(), "--key", KEY]) + .await? + .success()?; + expect_sandbox_setting(cx, &sandbox, UNSET, "unset").await +} + +async fn set_sandbox_setting( + cx: &Context, + sandbox: &Sandbox, + value: &str, +) -> testkit::Result { + cx.cli([ + "settings", + "set", + sandbox.name(), + "--key", + KEY, + "--value", + value, + ]) + .await +} + +/// Wait until `settings get` for the sandbox shows [`KEY`] with `value` from +/// `scope`. +async fn expect_sandbox_setting( + cx: &Context, + sandbox: &Sandbox, + value: &str, + scope: &str, +) -> testkit::Result { + poll_until( + &format!( + "{KEY} on sandbox {} to be {value} ({scope})", + sandbox.name() + ), + Duration::from_secs(10), + Duration::from_secs(1), + async || { + let settings: Value = cx + .cli(["settings", "get", sandbox.name(), "--json"]) + .await? + .success()? + .json()?; + let setting = &settings["settings"][KEY]; + Ok(if setting["value"] == value && setting["scope"] == scope { + Poll::Ready(()) + } else { + Poll::Pending(format!("{KEY} is {setting}")) + }) + }, + ) + .await +} + +/// The gateway-global value of `key` as the CLI shows it, [`UNSET`] if none. +async fn read_global_setting(cx: &Context, key: &str) -> testkit::Result { + let settings: Value = cx + .cli(["settings", "get", "--global", "--json"]) + .await? + .success()? + .json()?; + Ok(settings["settings"][key] + .as_str() + .unwrap_or(UNSET) + .to_string()) +} diff --git a/tests/suites/conformance/src/smoke.rs b/tests/suites/conformance/src/smoke.rs index d7255d44ea..eba5ed4995 100644 --- a/tests/suites/conformance/src/smoke.rs +++ b/tests/suites/conformance/src/smoke.rs @@ -12,25 +12,15 @@ async fn create_exec_delete(cx: &mut Context) -> testkit::Result { let sandbox = cx .sandbox("smoke") .command(["sleep", "infinity"]) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; let marker = format!("openshell-e2e-{}", cx.run_id()); - let output = sandbox.exec(["echo", marker.as_str()]).await?; - testkit::ensure!( - output.exit_code == 0, - "exec exited with {}; stderr: {}", - output.exit_code, - output.stderr - ); - testkit::ensure!( - output.stdout == format!("{marker}\n"), - "exec stdout was {:?}; expected {:?}", - output.stdout, - format!("{marker}\n") - ); + sandbox + .exec(["echo", marker.as_str()]) + .await? + .expect_stdout(&format!("{marker}\n"))?; sandbox.delete().await?; - cx.expect_absent(&sandbox).await + sandbox.expect_absent().await } diff --git a/tests/suites/conformance/src/templates.rs b/tests/suites/conformance/src/templates.rs new file mode 100644 index 0000000000..cfb9a309c7 --- /dev/null +++ b/tests/suites/conformance/src/templates.rs @@ -0,0 +1,178 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox templates suite: reusable workload templates and sandboxes created +//! from them. + +use std::collections::HashMap; + +use testkit::sdk::raw::proto::datamodel::v1::ObjectMeta; +use testkit::sdk::{ + DeleteOptions, SandboxResources, SandboxTemplateCreateSpec, SandboxTemplateListOptions, + SandboxWorkloadConfig, SandboxWorkloadTemplate, SandboxWorkloadTemplateSpec, SdkError, +}; +use testkit::{Context, ResultExt as _}; + +/// Create template `name`, and delete it after the test. The cleanup is +/// registered before the create, so it also covers a create the gateway +/// applied but reported as failed. +async fn create_template( + cx: &Context, + name: &str, + labels: HashMap, + workload: SandboxWorkloadConfig, +) -> Result { + let client = cx.client().clone(); + let owned = name.to_string(); + cx.on_cleanup(format!("delete sandbox template {name}"), async move || { + client + .delete_sandbox_template( + &owned, + DeleteOptions { + allow_missing: true, + }, + ) + .await?; + Ok(()) + }); + cx.client() + .create_sandbox_template(SandboxWorkloadTemplate { + metadata: Some(ObjectMeta { + name: name.to_string(), + labels, + ..ObjectMeta::default() + }), + spec: Some(SandboxWorkloadTemplateSpec { + workload: Some(workload), + ..SandboxWorkloadTemplateSpec::default() + }), + }) + .await +} + +/// A template stores its labels, environment and resources, is listed, and a +/// sandbox created from it runs with its environment and records which +/// template it came from. +#[testkit::test] +async fn sandbox_from_template_uses_its_workload(cx: &mut Context) -> testkit::Result { + let template_name = cx.name("tp"); + let labels = HashMap::from([("e2e".to_string(), "sandbox-template".to_string())]); + let workload = SandboxWorkloadConfig { + environment: HashMap::from([("FEATURE_FLAG".to_string(), "on".to_string())]), + resources: Some(SandboxResources { + cpu: "500m".to_string(), + memory: "512Mi".to_string(), + ..SandboxResources::default() + }), + ..SandboxWorkloadConfig::default() + }; + create_template(cx, &template_name, labels.clone(), workload.clone()).await?; + + let stored = cx.client().get_sandbox_template(&template_name).await?; + let meta = stored.metadata.clone().unwrap_or_default(); + testkit::ensure!( + meta.name == template_name && meta.labels.get("e2e") == labels.get("e2e"), + "template get returned name {:?} with labels {:?}; expected {template_name:?} with {labels:?}", + meta.name, + meta.labels + ); + let stored_workload = stored + .spec + .as_ref() + .and_then(|spec| spec.workload.clone()) + .unwrap_or_default(); + // Field by field: the gateway may fill in fields the template left unset. + let sent = workload.resources.clone().unwrap_or_default(); + let resources = stored_workload.resources.clone().unwrap_or_default(); + testkit::ensure!( + stored_workload.environment == workload.environment + && resources.cpu == sent.cpu + && resources.memory == sent.memory, + "template get returned workload {stored_workload:?}; expected environment {:?}, cpu {:?}, memory {:?}", + workload.environment, + sent.cpu, + sent.memory + ); + + let listed = cx + .client() + .list_all_sandbox_templates(SandboxTemplateListOptions::default()) + .await?; + testkit::ensure!( + listed.iter().any(|entry| entry + .metadata + .as_ref() + .is_some_and(|m| m.name == template_name)), + "template list does not include {template_name}" + ); + + // `SandboxBuilder` can't create from a template, so name, track and adopt by hand. + let sandbox_name = cx.name("ts"); + cx.track(&sandbox_name); + cx.client() + .create_sandbox_from_template(SandboxTemplateCreateSpec { + name: Some(sandbox_name.clone()), + template_name: template_name.clone(), + command: vec!["sleep".to_string(), "infinity".to_string()], + ..SandboxTemplateCreateSpec::default() + }) + .await?; + let sandbox = cx.adopt(&sandbox_name).await?; + let ready = sandbox.wait_ready().await?; + let provenance = ready.created_from_workload_template.as_ref(); + testkit::ensure!( + provenance.is_some_and(|p| p.name == template_name), + "sandbox {sandbox_name} records template {provenance:?}; expected {template_name:?}" + ); + + sandbox + .exec(["sh", "-c", "printf '%s' \"${FEATURE_FLAG:-missing}\""]) + .await? + .expect_stdout(&workload.environment["FEATURE_FLAG"]) + .context("FEATURE_FLAG in a sandbox from the template")?; + Ok(()) +} + +/// Template operations fail with the right status: a duplicate name is +/// `AlreadyExists`, a deleted template is `NotFound`, and creating a sandbox +/// from a missing template is `NotFound`. +#[testkit::test] +async fn template_crud_errors(cx: &mut Context) -> testkit::Result { + let name = cx.name("te"); + let empty = + async || create_template(cx, &name, HashMap::new(), SandboxWorkloadConfig::default()).await; + + empty().await?; + let again = empty().await; + testkit::ensure!( + matches!(again, Err(SdkError::AlreadyExists { .. })), + "creating template {name} again returned {again:?}; expected AlreadyExists" + ); + + cx.client() + .delete_sandbox_template(&name, DeleteOptions::default()) + .await?; + let deleted = cx.client().get_sandbox_template(&name).await; + testkit::ensure!( + matches!(deleted, Err(SdkError::NotFound { .. })), + "getting deleted template {name} returned {deleted:?}; expected NotFound" + ); + + let sandbox_name = cx.name("tm"); + cx.track(&sandbox_name); + let missing = cx.name("tx"); + let from_missing = cx + .client() + .create_sandbox_from_template(SandboxTemplateCreateSpec { + name: Some(sandbox_name), + template_name: missing.clone(), + command: vec!["true".to_string()], + ..SandboxTemplateCreateSpec::default() + }) + .await; + testkit::ensure!( + matches!(from_missing, Err(SdkError::NotFound { .. })), + "creating a sandbox from missing template {missing} returned {from_missing:?}; expected NotFound" + ); + Ok(()) +} diff --git a/tests/suites/drivers/podman/tests/oci_identity.rs b/tests/suites/drivers/podman/tests/oci_identity.rs new file mode 100644 index 0000000000..ae214ed283 --- /dev/null +++ b/tests/suites/drivers/podman/tests/oci_identity.rs @@ -0,0 +1,94 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Podman driver supervisor identity. +//! +//! Podman runs a sandbox's supervisor container as the workload's resolved +//! OCI identity with no effective capabilities; Docker runs it as a fixed +//! unprivileged user. The workload side of the same image is covered by the +//! `oci-image` feature suite in `tests/suites/features`. + +use container_engine_support::Engine; +use testkit::{Context, ResultExt as _}; + +const OCI_IDENTITY: &str = "2345:2346"; +/// A complete policy without a `process` section, so the sandbox identity +/// falls back to the image `USER`. +const IMAGE_IDENTITY_POLICY: &str = "version: 1 + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /lib64, /proc, /dev/urandom, /etc] + read_write: [/sandbox, /tmp, /dev/null] +landlock: + compatibility: best_effort + +network_policies: {} +"; + +/// The supervisor container runs as the image's numeric `USER`, not root or a +/// fixed user, and has no effective capabilities. +#[testkit::test] +async fn supervisor_runs_as_oci_identity_without_capabilities(cx: &mut Context) -> testkit::Result { + cx.expect_compute_driver("podman").await?; + let engine = Engine::from_env()?; + + let image = engine + .build_test_image( + cx, + "oci-identity", + &format!( + "FROM {} +USER 0:0 +RUN mkdir -p /home/app/project && \\ + chown {OCI_IDENTITY} /home/app /home/app/project && \\ + chmod 0700 /home/app /home/app/project +WORKDIR /home/app/project +USER {OCI_IDENTITY} +", + testkit::base_image() + ), + ) + .await?; + let image_user = engine + .output([ + "image", + "inspect", + "--format", + "{{.Config.User}}", + &image.tag, + ]) + .await?; + testkit::ensure!( + image_user == OCI_IDENTITY, + "the test image has USER {image_user:?}; expected {OCI_IDENTITY}" + ); + + let sandbox = cx + .cli_sandbox("oi") + .image(&image.tag) + .policy_yaml(IMAGE_IDENTITY_POLICY) + .command(["sleep", "infinity"]) + .create_ready() + .await + .context("sandbox starts from the OCI identity image")?; + + let supervisor = engine + .container_for_role(sandbox.name(), "supervisor") + .await?; + let user = engine + .inspect_container(&supervisor, "{{.Config.User}}") + .await?; + testkit::ensure!( + user == OCI_IDENTITY, + "the supervisor container runs as {user:?}; expected the image identity {OCI_IDENTITY}" + ); + let caps = engine + .inspect_container(&supervisor, "{{.EffectiveCaps}}") + .await?; + testkit::ensure!( + caps == "[]", + "the supervisor container has effective capabilities {caps}; expected none" + ); + Ok(()) +} diff --git a/tests/suites/drivers/podman/tests/userns.rs b/tests/suites/drivers/podman/tests/userns.rs index 5dff4a8a44..545554bfff 100644 --- a/tests/suites/drivers/podman/tests/userns.rs +++ b/tests/suites/drivers/podman/tests/userns.rs @@ -37,22 +37,11 @@ async fn configured_userns_matches_podman_reference(cx: &mut Context) -> testkit let image = testkit::base_image(); let reference = reference_uid_map(&engine, userns, &image).await?; - let name = cx.name("pu"); - cx.track(&name); let run = cx - .cli([ - "sandbox", - "create", - "--name", - &name, - "--from", - &image, - "--no-tty", - "--", - "sh", - "-c", - WORKSPACE_AND_UID_MAP_PROBE, - ]) + .cli_sandbox("pu") + .image(&image) + .command(["sh", "-c", WORKSPACE_AND_UID_MAP_PROBE]) + .run() .await? .success() .context("sandbox can write to its owned workspace and prints its UID map")?; diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 2db8da7663..0d91b3a2a9 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -2139,7 +2139,6 @@ version = "0.0.0" dependencies = [ "container-engine-support", "openshell-testkit", - "serde_json", "tempfile", ] diff --git a/tests/suites/features/cdi/src/lib.rs b/tests/suites/features/cdi/src/lib.rs index ca4e91f978..46a94f3c86 100644 --- a/tests/suites/features/cdi/src/lib.rs +++ b/tests/suites/features/cdi/src/lib.rs @@ -300,12 +300,10 @@ fn load(name: &str) -> Result { "{SCENARIOS_ENV} is not set; the environment must publish the scenario expectations" )) })?; - let text = std::fs::read_to_string(&path) - .map_err(Error::from_display) - .context(format!("read scenario expectations {path}"))?; - let document: Value = serde_json::from_str(&text) - .map_err(Error::from_display) - .context(format!("parse scenario expectations {path}"))?; + let text = + std::fs::read_to_string(&path).context(format!("read scenario expectations {path}"))?; + let document: Value = + serde_json::from_str(&text).context(format!("parse scenario expectations {path}"))?; scenario_expectations(&document, name) } diff --git a/tests/suites/features/host-gateway/tests/host_gateway.rs b/tests/suites/features/host-gateway/tests/host_gateway.rs index 472197dfd1..7d285116e4 100644 --- a/tests/suites/features/host-gateway/tests/host_gateway.rs +++ b/tests/suites/features/host-gateway/tests/host_gateway.rs @@ -11,7 +11,8 @@ use host_server_support::HttpRecorder; use testkit::policy; -use testkit::{Context, PYTHON_BINARIES, Result, ResultExt as _, YamlFile}; +use testkit::sdk::raw::proto; +use testkit::{Context, PYTHON_BINARIES, ResultExt as _}; /// The image's Python, which needs nothing beyond its standard library. The /// policy allows [`PYTHON_BINARIES`], the real paths behind `python3` in @@ -42,8 +43,6 @@ print(response.decode(errors="replace")) async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result { let marker = format!("host-gateway-{}", cx.run_id()); let server = HttpRecorder::start(marker.clone()).await?; - let policy = YamlFile::new(&policy(server.port)?)?; - let port = server.port.to_string(); let image = testkit::python_image(); // The client runs through `sandbox exec` in a Ready sandbox, not as the @@ -51,33 +50,17 @@ async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result // attach to a main process that already finished with "sandbox is not // ready" when the request reaches a replica that doesn't hold the // sandbox's session. - let name = cx.name("hg"); - cx.track(&name); - cx.cli([ - "sandbox", - "create", - "--name", - &name, - "--from", - &image, - "--policy", - policy.path()?, - "--detach", - "--no-tty", - "--", - "sh", - "-c", - "exec sleep infinity", - ]) - .await? - .success() - .context(format!("create a sandbox from {image}"))?; - cx.adopt(&name).await?.wait_ready().await?; + let sandbox = cx + .sandbox("hg") + .image(image.clone()) + .policy(policy(server.port)) + .command(["sh", "-c", "exec sleep infinity"]) + .create_ready() + .await + .context(format!("create a sandbox from {image}"))?; - let client = cx - .cli([ - "sandbox", "exec", "--name", &name, "--no-tty", "--", PYTHON, "-c", CLIENT, &port, - ]) + let client = sandbox + .exec([PYTHON, "-c", CLIENT, &server.port.to_string()]) .await? .success() .context("run the client in the sandbox")?; @@ -94,7 +77,7 @@ async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result /// connect to `host.openshell.internal` on `port`. The endpoint names only the /// host and port, as the policy docs describe, so the driver's address for the /// name is what makes it reachable, not an `allowed_ips` range. -fn policy(port: u16) -> Result { +fn policy(port: u16) -> proto::SandboxPolicy { let mut policy = policy::base(); policy::allow( &mut policy, @@ -103,5 +86,5 @@ fn policy(port: u16) -> Result { port, PYTHON_BINARIES, ); - policy::to_yaml(&policy) + policy } diff --git a/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs b/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs index 8035d0694e..a617c24843 100644 --- a/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs +++ b/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs @@ -9,7 +9,7 @@ use std::io::Write as _; use keycloak_support::KeycloakContext; -use testkit::{CliOutput, Context, Error, ResultExt as _}; +use testkit::{CliOutput, Context, ResultExt as _}; const CREDENTIAL_KEY: &str = "KEYCLOAK_ACCESS_TOKEN"; @@ -172,11 +172,9 @@ binaries: let mut file = tempfile::Builder::new() .suffix(".yaml") .tempfile() - .map_err(|error| Error::from_error(&error)) .context("create provider profile")?; file.write_all(profile.as_bytes()) .and_then(|()| file.flush()) - .map_err(|error| Error::from_error(&error)) .context("write provider profile")?; Ok(file) } diff --git a/tests/suites/features/kubernetes-updates/tests/update.rs b/tests/suites/features/kubernetes-updates/tests/update.rs index 61ac57e9fa..603059cd4e 100644 --- a/tests/suites/features/kubernetes-updates/tests/update.rs +++ b/tests/suites/features/kubernetes-updates/tests/update.rs @@ -458,10 +458,9 @@ async fn image_update_retains_workspace_and_identity(cx: &mut Context) -> testki let sandbox = cx // The testkit's unique prefix leaves five characters of the name limit. .sandbox("image") - .spec(|spec| spec.image = Some(ORIGINAL_IMAGE.to_string())) - .create() + .image(ORIGINAL_IMAGE.as_str()) + .create_ready() .await?; - sandbox.wait_ready().await?; let original = get_sandbox(&mut client, workspace, sandbox.name()).await; let id = &original.metadata.as_ref().expect("sandbox metadata").id; assert!(!id.is_empty(), "sandbox must have a canonical ID"); @@ -597,10 +596,9 @@ async fn terminal_workload_can_stop_update_and_restart(cx: &mut Context) -> test let sandbox = cx .sandbox("term") .restart_policy(testkit::sdk::SandboxRestartPolicy::Never) - .spec(|spec| spec.image = Some(ORIGINAL_IMAGE.to_string())) - .create() + .image(ORIGINAL_IMAGE.as_str()) + .create_ready() .await?; - sandbox.wait_ready().await?; let original = get_sandbox(&mut client, workspace, sandbox.name()).await; let before = sandbox_resource(sandbox.id()).await; let original_claims = claim_uids(&before).await; diff --git a/tests/suites/features/oci-image/Cargo.toml b/tests/suites/features/oci-image/Cargo.toml index b966d2c38d..4298cd0686 100644 --- a/tests/suites/features/oci-image/Cargo.toml +++ b/tests/suites/features/oci-image/Cargo.toml @@ -14,5 +14,4 @@ publish = false [dependencies] container-engine-support = { path = "../../../support/container-engine" } openshell-testkit = { path = "../../../testkit/testkit" } -serde_json = "1" tempfile = "3" diff --git a/tests/suites/features/oci-image/tests/oci_image.rs b/tests/suites/features/oci-image/tests/oci_image.rs index 4206c1ccb5..8e156eb8eb 100644 --- a/tests/suites/features/oci-image/tests/oci_image.rs +++ b/tests/suites/features/oci-image/tests/oci_image.rs @@ -11,7 +11,6 @@ //! driver created, through the same engine. use container_engine_support::{Engine, Image}; -use serde_json::Value; use testkit::{Context, Error, Result, ResultExt as _}; /// Sandbox condition reason for an image `WORKDIR` the sandbox identity @@ -87,10 +86,9 @@ USER 2345:2346 ), ) .await?; - let policy = PolicyFile::new(IMAGE_IDENTITY_POLICY)?; let checks = workspace_checks("2345:2346", "/home/app/project", true); let sandbox = oci - .create_sandbox("pu", &image, Some(policy.path()?), &checks) + .create_sandbox("pu", &image, Some(IMAGE_IDENTITY_POLICY), &checks) .await?; oci.container_checks(&sandbox, &image, "2345:2346").await }) @@ -106,13 +104,12 @@ async fn userless_image_uses_default_identity(cx: &mut Context) -> testkit::Resu let image = oci .build_image("userless", &format!("FROM {}\n", oci.base_image)) .await?; - let policy = PolicyFile::new(IMAGE_IDENTITY_POLICY)?; let checks = format!( "{} test \"$(id -G)\" = 1000;", workspace_checks("1000:1000", "/sandbox", false) ); let sandbox = oci - .create_sandbox("ud", &image, Some(policy.path()?), &checks) + .create_sandbox("ud", &image, Some(IMAGE_IDENTITY_POLICY), &checks) .await?; oci.container_checks(&sandbox, &image, "1000:1000").await }) @@ -137,19 +134,19 @@ USER root ), ) .await?; - let policy = PolicyFile::new(&format!( + let policy = format!( "{IMAGE_IDENTITY_POLICY} process: run_as_user: \"4234\" run_as_group: \"4235\" " - ))?; + ); let checks = format!( "{} test \"$(id -G)\" = 4235;", workspace_checks("4234:4235", "/sandbox", false) ); let sandbox = oci - .create_sandbox("pr", &image, Some(policy.path()?), &checks) + .create_sandbox("pr", &image, Some(&policy), &checks) .await?; oci.container_checks(&sandbox, &image, "4234:4235").await }) @@ -198,22 +195,12 @@ USER app ), ) .await?; - let name = oci.track(suffix); - let create = oci - .cx - .cli([ - "sandbox", - "create", - "--name", - &name, - "--from", - &image.tag, - "--no-tty", - "--", - "sh", - "-c", - "echo workspace-access-marker", - ]) + let builder = oci.cx.cli_sandbox(suffix); + let name = builder.name().to_string(); + let create = builder + .image(&image.tag) + .command(["sh", "-c", "echo workspace-access-marker"]) + .run() .await?; if writable { let create = create @@ -238,12 +225,7 @@ USER app ); // Require the condition reason, so another provisioning failure // cannot pass as a workspace rejection. - let sandbox: Value = oci - .cx - .cli(["sandbox", "get", &name, "--output", "json"]) - .await? - .success()? - .json()?; + let sandbox = oci.cx.adopt(&name).await?.get_json().await?; let reasons: Vec<&str> = sandbox["conditions"] .as_array() .into_iter() @@ -278,27 +260,12 @@ async fn run(cx: &Context, test: impl AsyncFnOnce(&Oci<'_>) -> Result) -> Result } impl Oci<'_> { - /// Build `containerfile` with a tag unique to this run, and remove the - /// image when the test ends. Cleanup runs last registered first, so the - /// sandboxes created from the image are deleted before it is removed. + /// Build `containerfile` as a test image named `oci-`, removed when + /// the test ends. async fn build_image(&self, name: &str, containerfile: &str) -> Result { - let tag = format!("localhost/openshell-test-oci-{name}:{}", self.cx.run_id()); - self.engine.build_image(&tag, containerfile).await?; - let (engine, built) = (self.engine.clone(), tag.clone()); - self.cx - .on_cleanup(format!("remove image {tag}"), async move || { - engine.remove_image(&built).await - }); - self.engine.image(&tag).await - } - - /// Name a sandbox for this test and register it for cleanup. Sandbox - /// names are limited to 19 characters on some drivers, so keep `short` - /// to a few characters. - fn track(&self, short: &str) -> String { - let name = self.cx.name(short); - self.cx.track(&name); - name + self.engine + .build_test_image(self.cx, &format!("oci-{name}"), containerfile) + .await } /// Create a detached sandbox whose main process runs `checks` and writes @@ -311,21 +278,21 @@ impl Oci<'_> { policy: Option<&str>, checks: &str, ) -> Result { - let name = self.track(short); let main = format!( "(set -eu; {checks} touch main-write) >/tmp/oci-main.log 2>&1; \ echo $? >/tmp/oci-main.status; exec sleep infinity" ); - let mut args = vec!["sandbox", "create", "--name", &name, "--from", &image.tag]; + let mut builder = self.cx.cli_sandbox(short).image(&image.tag); if let Some(policy) = policy { - args.extend(["--policy", policy]); + builder = builder.policy_yaml(policy); } - args.extend(["--detach", "--", "sh", "-c", &main]); - self.cx - .cli(args) - .await? - .success() - .context("sandbox starts from the test image")?; + let name = builder + .command(["sh", "-c", &main]) + .create() + .await + .context("sandbox starts from the test image")? + .name() + .to_string(); let script = format!( "set -eu; i=0; \ @@ -346,9 +313,7 @@ impl Oci<'_> { /// Upload and download default to paths relative to the image workspace. async fn file_transfer_uses_workspace(&self, sandbox: &str) -> Result { - let local = tempfile::tempdir() - .map_err(|error| Error::from_error(&error)) - .context("create temp dir")?; + let local = tempfile::tempdir().context("create temp dir")?; let upload = local.path().join("oci-transfer.txt"); write(&upload, "oci-transfer-ok")?; self.cx @@ -381,9 +346,7 @@ impl Oci<'_> { .await? .success() .context("download resolves relative paths in the workspace")?; - let downloaded = std::fs::read_to_string(&download) - .map_err(|error| Error::from_error(&error)) - .context("read download")?; + let downloaded = std::fs::read_to_string(&download).context("read download")?; testkit::ensure!( downloaded == "oci-transfer-ok", "downloaded file has unexpected content: {downloaded:?}" @@ -391,9 +354,7 @@ impl Oci<'_> { // A directory upload merges into an existing workspace directory. let merge = local.path().join("merge-upload"); - std::fs::create_dir(&merge) - .map_err(|error| Error::from_error(&error)) - .context("create merge dir")?; + std::fs::create_dir(&merge).context("create merge dir")?; write(&merge.join("conflict.txt"), "local-conflict")?; write(&merge.join("added.txt"), "local-added")?; self.exec( @@ -527,27 +488,8 @@ fn workspace_checks(identity: &str, workspace: &str, image_file: bool) -> String checks } -/// A sandbox policy written to a temporary file for `sandbox create --policy`. -struct PolicyFile(tempfile::NamedTempFile); - -impl PolicyFile { - fn new(contents: &str) -> Result { - let file = tempfile::NamedTempFile::new() - .map_err(|error| Error::from_error(&error)) - .context("create policy file")?; - write(file.path(), contents)?; - Ok(Self(file)) - } - - fn path(&self) -> Result<&str> { - utf8(self.0.path()) - } -} - fn write(path: &std::path::Path, contents: &str) -> Result { - std::fs::write(path, contents) - .map_err(|error| Error::from_error(&error)) - .context(format!("write {}", path.display())) + std::fs::write(path, contents).context(format!("write {}", path.display())) } fn utf8(path: &std::path::Path) -> Result<&str> { diff --git a/tests/support/container-engine/src/lib.rs b/tests/support/container-engine/src/lib.rs index f380a9815f..7d47f2d33f 100644 --- a/tests/support/container-engine/src/lib.rs +++ b/tests/support/container-engine/src/lib.rs @@ -20,7 +20,7 @@ use std::path::Path; -use testkit::{Error, Result, ResultExt as _}; +use testkit::{Context, Error, Result, ResultExt as _}; use tokio::process::Command; pub const ENGINE_ENV: &str = "OPENSHELL_TEST_CONTAINER_ENGINE"; @@ -64,17 +64,11 @@ impl Engine { Ok(Self { command }) } - /// Build `containerfile` as `tag`. The build context is empty. Remove - /// the image with [`Engine::remove_image`], for example in a cleanup - /// hook registered right after the build. - pub async fn build_image(&self, tag: &str, containerfile: &str) -> Result { - let context = tempfile::tempdir() - .map_err(|error| Error::from_error(&error)) - .context("create build context")?; + /// Build `containerfile` as `tag`. The build context is empty. + async fn build_image(&self, tag: &str, containerfile: &str) -> Result { + let context = tempfile::tempdir().context("create build context")?; let file = context.path().join("Containerfile"); - std::fs::write(&file, containerfile) - .map_err(|error| Error::from_error(&error)) - .context("write Containerfile")?; + std::fs::write(&file, containerfile).context("write Containerfile")?; self.output([ "build", "--file", @@ -87,6 +81,25 @@ impl Engine { .map(drop) } + /// Build `containerfile` as `localhost/openshell-test-:` + /// and remove the image when the test ends. Cleanup runs last registered + /// first, so sandboxes created from the image afterwards are deleted + /// before it is removed. + pub async fn build_test_image( + &self, + cx: &Context, + name: &str, + containerfile: &str, + ) -> Result { + let tag = format!("localhost/openshell-test-{name}:{}", cx.run_id()); + self.build_image(&tag, containerfile).await?; + let (engine, built) = (self.clone(), tag.clone()); + cx.on_cleanup(format!("remove image {tag}"), async move || { + engine.remove_image(&built).await + }); + self.image(&tag).await + } + /// Image `tag` and the ID it has now. pub async fn image(&self, tag: &str) -> Result { let id = self @@ -157,7 +170,6 @@ impl Engine { .kill_on_drop(true) .output() .await - .map_err(|error| Error::from_error(&error)) .context(format!("run {command}"))?; testkit::ensure!( output.status.success(), diff --git a/tests/support/host-server/src/lib.rs b/tests/support/host-server/src/lib.rs index 0c11e4ceca..8ac8528570 100644 --- a/tests/support/host-server/src/lib.rs +++ b/tests/support/host-server/src/lib.rs @@ -15,7 +15,7 @@ mod websocket; use std::time::Duration; -use testkit::{Error, Result, ResultExt as _}; +use testkit::{Result, ResultExt as _}; use tokio::net::TcpListener; pub use http::{HttpRecorder, Request}; @@ -29,11 +29,9 @@ const MAX_REQUEST: usize = 1024 * 1024; async fn bind() -> Result<(TcpListener, u16)> { let listener = TcpListener::bind(("0.0.0.0", 0)) .await - .map_err(|error| Error::from_error(&error)) .context("bind a host server")?; let port = listener .local_addr() - .map_err(|error| Error::from_error(&error)) .context("read the host server's address")? .port(); Ok((listener, port)) diff --git a/tests/support/keycloak/src/lib.rs b/tests/support/keycloak/src/lib.rs index d5b0b58f31..468cf29278 100644 --- a/tests/support/keycloak/src/lib.rs +++ b/tests/support/keycloak/src/lib.rs @@ -158,7 +158,6 @@ impl Keycloak { .kill_on_drop(true) .output() .await - .map_err(|error| Error::from_error(&error)) .context("run curl")?; Ok(if output.status.success() { Poll::Ready(()) @@ -186,9 +185,8 @@ impl Keycloak { .await .context("Keycloak password grant")?; - let response: Value = serde_json::from_slice(&stdout) - .map_err(|error| Error::from_error(&error)) - .context("decode Keycloak grant response")?; + let response: Value = + serde_json::from_slice(&stdout).context("decode Keycloak grant response")?; let token = |name: &str| { response .get(name) @@ -253,20 +251,14 @@ async fn post_form(url: &str, fields: &[&str], secret: (&str, &str)) -> Result Result { - crate::ensure!( - self.exit_code == 0, - "`{}` exited with {}; stderr: {}", - self.command, - self.exit_code, - self.stderr.trim() - ); + self.check(|outcome| outcome.success())?; Ok(self) } /// Fail unless the command exited non-zero, e.g. when a request must be /// rejected. pub fn failure(self) -> Result { - crate::ensure!( - self.exit_code != 0, - "`{}` succeeded but was expected to fail; stdout: {}", - self.command, - self.stdout.trim() - ); + self.check(|outcome| outcome.failure())?; + Ok(self) + } + + /// Fail unless the command exited 0 and printed exactly `expected`. + pub fn expect_stdout(self, expected: &str) -> Result { + self.check(|outcome| outcome.stdout_is(expected))?; Ok(self) } + fn check(&self, check: impl FnOnce(&Outcome<'_>) -> Result) -> Result { + let what = format!("`{}`", self.command); + check(&Outcome { + what: &what, + exit_code: self.exit_code, + stdout: &self.stdout, + stderr: &self.stderr, + }) + } + + /// Stdout and stderr as one line of words, for matching an error report. + /// The CLI wraps reports to the terminal width and draws a gutter, so + /// match against this rather than the raw output. + pub fn report_text(&self) -> String { + format!("{} {}", self.stdout, self.stderr) + .replace(REPORT_GUTTER, " ") + .split_whitespace() + .collect::>() + .join(" ") + } + /// Parse stdout as JSON, from a command run with `--output json`. pub fn json(&self) -> Result { serde_json::from_str(&self.stdout) - .map_err(|error| Error::from_error(&error)) .context(format!("parse JSON output of `{}`", self.command)) } } diff --git a/tests/testkit/testkit/src/cli_sandbox.rs b/tests/testkit/testkit/src/cli_sandbox.rs new file mode 100644 index 0000000000..df645a86fd --- /dev/null +++ b/tests/testkit/testkit/src/cli_sandbox.rs @@ -0,0 +1,115 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandboxes created through `openshell sandbox create`, for tests of what +//! only the CLI offers, such as `--policy` files, `--no-keep`, or an attached +//! main command. + +use crate::cli::CliOutput; +use crate::context::{Handles, Sandbox}; +use crate::error::{Result, ResultExt}; +use crate::yaml_file::YamlFile; + +/// A sandbox to create with `openshell sandbox create`. Start one with +/// [`Context::cli_sandbox`](crate::Context::cli_sandbox). +/// +/// The sandbox is tracked for cleanup before the CLI runs, so a create the +/// gateway accepted but the CLI reported as failed is still cleaned up. The +/// CLI runs with `--no-tty`: tests never have a terminal. +#[must_use = "a CliSandboxBuilder does nothing until create(), create_ready() or run()"] +pub struct CliSandboxBuilder { + handles: Handles, + name: String, + args: Vec, + policy: Option, + command: Vec, +} + +impl CliSandboxBuilder { + pub(crate) fn new(handles: Handles, name: String) -> Self { + Self { + handles, + name, + args: Vec::new(), + policy: None, + command: Vec::new(), + } + } + + /// The sandbox's name, for building the command before the create. + pub fn name(&self) -> &str { + &self.name + } + + /// Create the sandbox from `image` (`--from`). + pub fn image(self, image: impl Into) -> Self { + self.args(["--from".to_string(), image.into()]) + } + + /// Create the sandbox with this policy YAML (`--policy`). It is written + /// to a temporary file that lives until the CLI exits. + pub fn policy_yaml(mut self, yaml: impl Into) -> Self { + self.policy = Some(yaml.into()); + self + } + + /// Other `sandbox create` arguments, such as `--no-keep` or `--label`. + pub fn args(mut self, args: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.args.extend(args.into_iter().map(Into::into)); + self + } + + /// The sandbox's main command. Empty selects the gateway's default. + pub fn command(mut self, command: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.command = command.into_iter().map(Into::into).collect(); + self + } + + /// Create the sandbox with `--detach`, require the CLI to succeed, and + /// return the sandbox. It need not be Ready yet. + pub async fn create(mut self) -> Result { + self.args.push("--detach".to_string()); + self.invoke() + .await? + .success() + .context(format!("create sandbox {}", self.name))?; + self.handles.adopt(&self.name).await + } + + /// Create the sandbox and wait until it is Ready. + pub async fn create_ready(self) -> Result { + let sandbox = self.create().await?; + sandbox.wait_ready().await?; + Ok(sandbox) + } + + /// Create the sandbox attached to its main command and return the CLI's + /// output, whatever its exit code. For a test of the main command's + /// result, or of a create that must fail. + pub async fn run(self) -> Result { + self.invoke().await + } + + async fn invoke(&self) -> Result { + self.handles.track(&self.name); + let policy = self.policy.as_deref().map(YamlFile::new).transpose()?; + let mut args = vec!["sandbox", "create", "--name", &self.name, "--no-tty"]; + if let Some(policy) = &policy { + args.extend(["--policy", policy.path()?]); + } + args.extend(self.args.iter().map(String::as_str)); + if !self.command.is_empty() { + args.push("--"); + args.extend(self.command.iter().map(String::as_str)); + } + self.handles.cli().run(args).await + } +} diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index 7f09ad7397..47221f68ec 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -14,7 +14,9 @@ use rand::Rng; use crate::cleanup::{CLEANUP_TIMEOUT, Cleanup, Key}; use crate::cli::{Cli, CliOutput}; +use crate::cli_sandbox::CliSandboxBuilder; use crate::error::{Error, Result, ResultExt}; +use crate::output::Outcome; use crate::poll::{Poll, poll_until}; /// Sandbox creation includes image pulls on a cold environment. @@ -27,24 +29,23 @@ const STATUS_INTERVAL: Duration = Duration::from_secs(1); /// What a test uses to act on the system under test. pub struct Context { - client: WorkspaceScopedClient, - grpc: Grpc, - cli: Cli, + handles: Handles, run_id: String, - cleanup: Cleanup, } impl Context { pub(crate) fn new(client: &OpenShellClient, workspace: &str, cli: Cli) -> Self { Self { - client: client.workspace(workspace), - grpc: Grpc { - client: client.clone(), - workspace: workspace.to_string(), + handles: Handles { + client: client.workspace(workspace), + grpc: Grpc { + client: client.clone(), + workspace: workspace.to_string(), + }, + cli, + cleanup: Cleanup::default(), }, - cli, run_id: generate_run_id(), - cleanup: Cleanup::default(), } } @@ -55,7 +56,7 @@ impl Context { I: IntoIterator, S: AsRef, { - self.cli.run(args).await + self.handles.cli.run(args).await } /// Like [`Context::cli`], with `dir` as the working directory. @@ -64,7 +65,7 @@ impl Context { I: IntoIterator, S: AsRef, { - self.cli.run_in(Some(dir), args).await + self.handles.cli.run_in(Some(dir), args).await } /// Like [`Context::cli`], with extra environment variables for the CLI. @@ -75,13 +76,13 @@ impl Context { I: IntoIterator, S: AsRef, { - self.cli.run_with_env(None, env, args).await + self.handles.cli.run_with_env(None, env, args).await } /// The CLI that [`Context::cli`] runs, for code that can't borrow the /// context, such as a cleanup hook. pub fn cli_handle(&self) -> Cli { - self.cli.clone() + self.handles.cli.clone() } /// The ID unique to this test run, used in resource names. @@ -91,7 +92,7 @@ impl Context { /// The SDK client, for operations testkit does not wrap. pub fn client(&self) -> &WorkspaceScopedClient { - &self.client + &self.handles.client } /// A resource name unique to this run: `test--`. @@ -102,10 +103,8 @@ impl Context { /// Start describing a sandbox named `cx.name(suffix)`. pub fn sandbox(&self, suffix: &str) -> SandboxBuilder { SandboxBuilder { - client: self.client.clone(), - grpc: self.grpc.clone(), + handles: self.handles.clone(), policy: None, - cleanup: self.cleanup.clone(), spec: SandboxSpec { name: Some(self.name(suffix)), ..SandboxSpec::default() @@ -113,6 +112,14 @@ impl Context { } } + /// Start describing a sandbox named `cx.name(suffix)` that the + /// `openshell` CLI creates. Use it for what only `sandbox create` + /// offers, such as a `--policy` file or `--no-keep`; prefer + /// [`Context::sandbox`] otherwise. + pub fn cli_sandbox(&self, suffix: &str) -> CliSandboxBuilder { + CliSandboxBuilder::new(self.handles.clone(), self.name(suffix)) + } + /// Track a sandbox the test is about to create through the CLI, so cleanup /// deletes it even if the test fails after the gateway accepted the create. /// Call this before the CLI command; use [`Context::adopt`] afterwards. @@ -121,7 +128,7 @@ impl Context { /// in the cleanup order where it was first tracked. Tracking a sandbox /// again, for example by adopting it, doesn't move it. pub fn track(&self, name: &str) { - track_sandbox(&self.cleanup, &self.client, name); + self.handles.track(name); } /// Run `hook` after the test body, whether the body passes, fails, or @@ -158,50 +165,13 @@ impl Context { F: FnOnce() -> Fut + Send + 'static, Fut: Future + Send + 'static, { - self.cleanup.push(name, hook); + self.handles.cleanup.push(name, hook); } /// A handle to a sandbox created outside the builder, e.g. by the CLI. /// It is tracked for cleanup like a built one. pub async fn adopt(&self, name: &str) -> Result { - self.track(name); - let found = self - .client - .get_sandbox(name) - .await - .context(format!("get sandbox {name}"))?; - Ok(Sandbox { - client: self.client.clone(), - grpc: self.grpc.clone(), - cleanup: self.cleanup.clone(), - name: name.to_string(), - id: found.id, - }) - } - - /// Fail unless the sandbox is gone from both get and list. - pub async fn expect_absent(&self, sandbox: &Sandbox) -> Result { - let name = sandbox.name(); - match self.client.get_sandbox(name).await { - Err(SdkError::NotFound { .. }) => {} - Err(error) => return Err(Error::from(error).context(format!("get sandbox {name}"))), - Ok(found) => { - return Err(Error::msg(format!( - "sandbox {name} still exists (id {}, phase {:?})", - found.id, found.phase - ))); - } - } - let listed = self - .client - .list_all_sandboxes(ListOptions::default()) - .await - .context("list sandboxes")?; - crate::ensure!( - !listed.iter().any(|entry| entry.name == name), - "sandbox {name} is absent from get but still listed" - ); - Ok(()) + self.handles.adopt(name).await } /// Fail unless the gateway runs exactly one compute driver, `name`, as @@ -209,12 +179,12 @@ impl Context { /// pointed at another driver's gateway fails instead of testing it. pub async fn expect_compute_driver(&self, name: &str) -> Result { let info = self + .handles .grpc .client() .await? .get_gateway_info(proto::GetGatewayInfoRequest {}) .await - .map_err(|status| Error::from_error(&status)) .context("get gateway info")? .into_inner(); let drivers: Vec<(&str, &str)> = info @@ -235,30 +205,11 @@ impl Context { Ok(()) } - /// Fail unless `list` shows the sandbox in `phase`. - pub async fn expect_listed(&self, sandbox: &Sandbox, phase: SandboxPhase) -> Result { - let name = sandbox.name(); - let listed = self - .client - .list_all_sandboxes(ListOptions::default()) - .await - .context("list sandboxes")?; - let entry = listed.iter().find(|entry| entry.name == name); - crate::ensure!(entry.is_some(), "sandbox {name} is not listed"); - let entry = entry.expect("checked above"); - crate::ensure!( - entry.phase == phase, - "sandbox {name} is listed in phase {:?}; expected {phase:?}", - entry.phase - ); - Ok(()) - } - /// Run the cleanup hooks, including deleting the sandboxes the test /// created and has not deleted. Returns the cleanup failures; they are /// reported separately from the test result. pub(crate) async fn cleanup(&self) -> Vec { - self.cleanup.run(CLEANUP_TIMEOUT).await + self.handles.cleanup.run(CLEANUP_TIMEOUT).await } } @@ -304,18 +255,50 @@ impl Grpc { } } -/// A sandbox to create. -pub struct SandboxBuilder { +/// What the context, its builders and its sandboxes share: the gateway +/// clients, the CLI, and the test's cleanup list. +#[derive(Clone)] +pub(crate) struct Handles { client: WorkspaceScopedClient, grpc: Grpc, - policy: Option, + cli: Cli, cleanup: Cleanup, +} + +impl Handles { + pub(crate) fn cli(&self) -> &Cli { + &self.cli + } + + pub(crate) fn track(&self, name: &str) { + track_sandbox(&self.cleanup, &self.client, name); + } + + pub(crate) async fn adopt(&self, name: &str) -> Result { + self.track(name); + let found = self + .client + .get_sandbox(name) + .await + .context(format!("get sandbox {name}"))?; + Ok(Sandbox { + handles: self.clone(), + name: name.to_string(), + id: found.id, + }) + } +} + +/// A sandbox to create. +#[must_use = "a SandboxBuilder does nothing until create() or create_ready()"] +pub struct SandboxBuilder { + handles: Handles, + policy: Option, spec: SandboxSpec, } impl SandboxBuilder { /// The sandbox's main command. Empty selects the gateway's default. - #[must_use] pub fn command(mut self, command: I) -> Self where I: IntoIterator, @@ -325,35 +308,48 @@ impl SandboxBuilder { self } - #[must_use] + /// Create the sandbox from `image` instead of the gateway's default. + pub fn image(mut self, image: impl Into) -> Self { + self.spec.image = Some(image.into()); + self + } + + /// Whether the sandbox restarts when its main command exits: never, on + /// failure, or always. pub fn restart_policy(mut self, policy: SandboxRestartPolicy) -> Self { self.spec.restart_policy = policy; self } /// Create the sandbox with this policy instead of the gateway's default. - #[must_use] pub fn policy(mut self, policy: proto::SandboxPolicy) -> Self { self.policy = Some(policy); self } /// Full access to the spec for fields without a builder method. - #[must_use] pub fn spec(mut self, update: impl FnOnce(&mut SandboxSpec)) -> Self { update(&mut self.spec); self } + /// Create the sandbox and wait until it is Ready. + pub async fn create_ready(self) -> Result { + let sandbox = self.create().await?; + sandbox.wait_ready().await?; + Ok(sandbox) + } + /// Create the sandbox. It is tracked for cleanup before the request is /// sent, so a create that fails after the gateway accepted it is still /// cleaned up. pub async fn create(self) -> Result { let name = self.spec.name.clone().unwrap_or_default(); - track_sandbox(&self.cleanup, &self.client, &name); + self.handles.track(&name); let id = match self.policy { None => { - self.client + self.handles + .client .create_sandbox(self.spec) .await .context(format!("create sandbox {name}"))? @@ -365,15 +361,16 @@ impl SandboxBuilder { !self.spec.gpu && self.spec.service_exposures.is_empty(), "testkit doesn't create a sandbox with a policy and a GPU or service exposures" ); - let request = policy_create_request(self.spec, policy, self.grpc.scope()); - self.grpc + let request = policy_create_request(self.spec, policy, self.handles.grpc.scope()); + self.handles + .grpc .client() .await? .create_sandbox(request) .await - .map_err(|status| Error::from_status(&status)) .context(format!("create sandbox {name}"))?; - self.client + self.handles + .client .get_sandbox(&name) .await .context(format!("get sandbox {name}"))? @@ -381,9 +378,7 @@ impl SandboxBuilder { } }; Ok(Sandbox { - client: self.client, - grpc: self.grpc, - cleanup: self.cleanup, + handles: self.handles, name, id, }) @@ -436,9 +431,7 @@ fn policy_create_request( /// A sandbox created by the test. pub struct Sandbox { - client: WorkspaceScopedClient, - grpc: Grpc, - cleanup: Cleanup, + handles: Handles, name: String, id: String, } @@ -446,6 +439,8 @@ pub struct Sandbox { /// The buffered output of an exec. #[derive(Debug)] pub struct ExecOutput { + /// The command and the sandbox it ran in, for reports. + pub description: String, pub exit_code: i32, pub stdout: String, pub stderr: String, @@ -454,15 +449,31 @@ pub struct ExecOutput { impl ExecOutput { /// Fail unless the command exited 0. pub fn success(self) -> Result { - crate::ensure!( - self.exit_code == 0, - "exec exited with {}; stdout: {}; stderr: {}", - self.exit_code, - self.stdout.trim(), - self.stderr.trim() - ); + self.outcome().success()?; + Ok(self) + } + + /// Fail unless the command exited non-zero, e.g. when a request must be + /// rejected. + pub fn failure(self) -> Result { + self.outcome().failure()?; + Ok(self) + } + + /// Fail unless the command exited 0 and printed exactly `expected`. + pub fn expect_stdout(self, expected: &str) -> Result { + self.outcome().stdout_is(expected)?; Ok(self) } + + fn outcome(&self) -> Outcome<'_> { + Outcome { + what: &self.description, + exit_code: self.exit_code, + stdout: &self.stdout, + stderr: &self.stderr, + } + } } impl Sandbox { @@ -478,6 +489,7 @@ impl Sandbox { /// instead is an error. pub async fn wait_ready(&self) -> Result { let snapshot = self + .handles .client .wait_ready(&self.name, READY_TIMEOUT) .await @@ -512,7 +524,8 @@ impl Sandbox { /// Stop the sandbox and wait until it is Stopped. pub async fn stop(&self) -> Result { - self.client + self.handles + .client .stop_sandbox(&self.name) .await .context(format!("stop sandbox {}", self.name))?; @@ -521,7 +534,8 @@ impl Sandbox { /// Start the sandbox and wait until it is Ready again. pub async fn start(&self) -> Result { - self.client + self.handles + .client .start_sandbox(&self.name) .await .context(format!("start sandbox {}", self.name))?; @@ -550,17 +564,17 @@ impl Sandbox { pub async fn set_setting(&self, key: &str, value: proto::setting_value::Value) -> Result { let request = proto::UpdateConfigRequest { sandbox: self.name.clone(), - workspace_scope: self.grpc.scope(), + workspace_scope: self.handles.grpc.scope(), setting_key: key.to_string(), setting_value: Some(proto::SettingValue { value: Some(value) }), ..proto::UpdateConfigRequest::default() }; - self.grpc + self.handles + .grpc .client() .await? .update_config(request) .await - .map_err(|status| Error::from_error(&status)) .context(format!("set {key} on sandbox {}", self.name))?; Ok(()) } @@ -569,15 +583,15 @@ impl Sandbox { pub async fn config(&self) -> Result { let request = proto::GetSandboxConfigRequest { name: self.name.clone(), - workspace_scope: self.grpc.scope(), + workspace_scope: self.handles.grpc.scope(), }; Ok(self + .handles .grpc .client() .await? .get_sandbox_config(request) .await - .map_err(|status| Error::from_error(&status)) .context(format!("get config of sandbox {}", self.name))? .into_inner()) } @@ -586,23 +600,24 @@ impl Sandbox { pub async fn draft_policy(&self) -> Result { let request = proto::GetDraftPolicyRequest { sandbox: self.name.clone(), - workspace_scope: self.grpc.scope(), + workspace_scope: self.handles.grpc.scope(), status_filter: String::new(), }; Ok(self + .handles .grpc .client() .await? .get_draft_policy(request) .await - .map_err(|status| Error::from_error(&status)) .context(format!("get draft policy of sandbox {}", self.name))? .into_inner()) } /// The sandbox's current state. pub async fn get(&self) -> Result { - self.client + self.handles + .client .get_sandbox(&self.name) .await .context(format!("get sandbox {}", self.name)) @@ -615,33 +630,141 @@ impl Sandbox { S: Into, { let command: Vec = command.into_iter().map(Into::into).collect(); + let description = format!("exec {command:?} in sandbox {}", self.name); let options = ExecOptions { timeout: Some(EXEC_TIMEOUT), ..ExecOptions::default() }; let result = self + .handles .client .exec(&self.name, &command, options) .await - .context(format!("exec {command:?} in sandbox {}", self.name))?; + .context(&description)?; Ok(ExecOutput { + description, exit_code: result.exit_code, stdout: String::from_utf8_lossy(&result.stdout).into_owned(), stderr: String::from_utf8_lossy(&result.stderr).into_owned(), }) } + /// Run `command` every 250 ms until `ready` accepts its output, and return + /// that output. An exec error fails at once; a timeout reports the last + /// output. + pub async fn poll_exec( + &self, + what: &str, + timeout: Duration, + command: I, + ready: impl Fn(&ExecOutput) -> bool, + ) -> Result + where + I: IntoIterator, + S: Into, + { + let command: Vec = command.into_iter().map(Into::into).collect(); + poll_until(what, timeout, Duration::from_millis(250), async || { + let output = self.exec(command.iter().cloned()).await?; + Ok(if ready(&output) { + Poll::Ready(output) + } else { + Poll::Pending(format!( + "exited {} with stdout {:?}; stderr: {}", + output.exit_code, + output.stdout, + output.stderr.trim() + )) + }) + }) + .await + } + + /// `sandbox get --output json`. It reports fields the SDK does not + /// expose, such as `restart_count`, `restart_policy`, + /// `main_process_instance_id`, and `conditions`. + pub async fn get_json(&self) -> Result { + self.handles + .cli + .run(["sandbox", "get", &self.name, "--output", "json"]) + .await? + .success()? + .json() + } + + /// Fail unless `list` shows the sandbox in `phase`. + pub async fn expect_listed(&self, phase: SandboxPhase) -> Result { + let name = &self.name; + let listed = self + .handles + .client + .list_all_sandboxes(ListOptions::default()) + .await + .context("list sandboxes")?; + let Some(entry) = listed.iter().find(|entry| entry.name == *name) else { + return Err(Error::msg(format!("sandbox {name} is not listed"))); + }; + crate::ensure!( + entry.phase == phase, + "sandbox {name} is listed in phase {:?}; expected {phase:?}", + entry.phase + ); + Ok(()) + } + + /// Fail unless the sandbox is gone from both get and list. + pub async fn expect_absent(&self) -> Result { + let name = &self.name; + match self.handles.client.get_sandbox(name).await { + Err(SdkError::NotFound { .. }) => {} + Err(error) => return Err(Error::from(error).context(format!("get sandbox {name}"))), + Ok(found) => { + return Err(Error::msg(format!( + "sandbox {name} still exists (id {}, phase {:?})", + found.id, found.phase + ))); + } + } + let listed = self + .handles + .client + .list_all_sandboxes(ListOptions::default()) + .await + .context("list sandboxes")?; + crate::ensure!( + !listed.iter().any(|entry| entry.name == *name), + "sandbox {name} is absent from get but still listed" + ); + Ok(()) + } + + /// Wait up to `timeout` for the gateway to remove the sandbox by itself, + /// for example after a `--no-keep` main command exits, then check it is + /// gone from both get and list. + pub async fn wait_absent(&self, timeout: Duration) -> Result { + self.handles + .client + .wait_deleted(&self.name, timeout, Some(&self.id)) + .await + .context(format!("wait for sandbox {} to be removed", self.name))?; + self.expect_absent().await + } + /// Delete the sandbox and wait until it is gone. pub async fn delete(&self) -> Result { - self.client + self.handles + .client .delete_sandbox(&self.name, DeleteOptions::default()) .await .context(format!("delete sandbox {}", self.name))?; - self.client + self.handles + .client .wait_deleted(&self.name, DELETE_TIMEOUT, Some(&self.id)) .await .context(format!("wait for sandbox {} to be deleted", self.name))?; - self.cleanup.remove(&Key::Sandbox(self.name.clone())); + self.handles + .cleanup + .remove(&Key::Sandbox(self.name.clone())); Ok(()) } } diff --git a/tests/testkit/testkit/src/error.rs b/tests/testkit/testkit/src/error.rs index 51d2fc3904..18ce6138c7 100644 --- a/tests/testkit/testkit/src/error.rs +++ b/tests/testkit/testkit/src/error.rs @@ -26,26 +26,18 @@ impl Error { } } - /// A gateway's refusal, keeping its status code for [`Error::grpc_code`]. - pub fn from_status(status: &tonic::Status) -> Self { - Self { - code: Some(status.code()), - ..Self::from_error(status) - } - } - /// The gRPC status code, when the gateway refused a request. pub fn grpc_code(&self) -> Option { self.code } - pub fn from_display(error: impl fmt::Display) -> Self { + pub(crate) fn from_display(error: impl fmt::Display) -> Self { Self::msg(error.to_string()) } /// An error with its sources, so a wrapper such as tonic's /// "transport error" still says what went wrong underneath. - pub fn from_error(error: &(dyn std::error::Error + 'static)) -> Self { + pub(crate) fn from_error(error: &(dyn std::error::Error + 'static)) -> Self { let mut message = error.to_string(); let mut source = error.source(); while let Some(cause) = source { @@ -85,6 +77,34 @@ impl From for Error { } } +/// A gateway's refusal, keeping its status code for [`Error::grpc_code`]. +impl From for Error { + fn from(status: tonic::Status) -> Self { + Self { + code: Some(status.code()), + ..Self::from_error(&status) + } + } +} + +impl From for Error { + fn from(error: serde_json::Error) -> Self { + Self::from_error(&error) + } +} + +impl From for Error { + fn from(error: serde_yml::Error) -> Self { + Self::from_error(&error) + } +} + +impl From for Error { + fn from(error: std::io::Error) -> Self { + Self::from_error(&error) + } +} + /// Add context to a failing step. pub trait ResultExt { fn context(self, context: impl fmt::Display) -> Result; diff --git a/tests/testkit/testkit/src/gateway.rs b/tests/testkit/testkit/src/gateway.rs index 6513fcfc4e..fef05833b7 100644 --- a/tests/testkit/testkit/src/gateway.rs +++ b/tests/testkit/testkit/src/gateway.rs @@ -231,9 +231,7 @@ fn env(name: &str) -> Option { } fn read(path: &Path) -> Result> { - std::fs::read(path) - .map_err(Error::from_display) - .context(format!("read {}", path.display())) + std::fs::read(path).context(format!("read {}", path.display())) } #[cfg(test)] diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 8520fb80b3..86f28e2599 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -28,10 +28,12 @@ mod cleanup; mod cli; +mod cli_sandbox; mod context; mod error; pub mod gateway; mod image; +mod output; pub mod policy; mod poll; pub mod program; @@ -43,6 +45,7 @@ mod yaml_file; pub use cleanup::CLEANUP_TIMEOUT; pub use cli::{CLI_TIMEOUT, Cli, CliOutput}; +pub use cli_sandbox::CliSandboxBuilder; pub use context::{ Context, DELETE_TIMEOUT, EXEC_TIMEOUT, ExecOutput, READY_TIMEOUT, STOP_TIMEOUT, Sandbox, SandboxBuilder, @@ -51,7 +54,7 @@ pub use error::{Error, Result, ResultExt}; pub use gateway::Gateway; pub use image::{PYTHON_BINARIES, base_image, python_image}; pub use openshell_sdk as sdk; -pub use poll::{Poll, poll_until}; +pub use poll::{Poll, poll_until, with_deadline}; pub use program::main; pub use ready::check as check_ready; pub use registry::Test; diff --git a/tests/testkit/testkit/src/output.rs b/tests/testkit/testkit/src/output.rs new file mode 100644 index 0000000000..a0f91c2f63 --- /dev/null +++ b/tests/testkit/testkit/src/output.rs @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The checks [`ExecOutput`](crate::ExecOutput) and +//! [`CliOutput`](crate::CliOutput) share, so both report failures the same +//! way. + +use crate::error::Result; + +/// A finished command, as the checks see it. +pub(crate) struct Outcome<'a> { + /// The command, as failure messages name it. + pub(crate) what: &'a str, + pub(crate) exit_code: i32, + pub(crate) stdout: &'a str, + pub(crate) stderr: &'a str, +} + +impl Outcome<'_> { + pub(crate) fn success(&self) -> Result { + crate::ensure!( + self.exit_code == 0, + "{} exited with {}; stdout: {}; stderr: {}", + self.what, + self.exit_code, + self.stdout.trim(), + self.stderr.trim() + ); + Ok(()) + } + + pub(crate) fn failure(&self) -> Result { + crate::ensure!( + self.exit_code != 0, + "{} succeeded but was expected to fail; stdout: {}; stderr: {}", + self.what, + self.stdout.trim(), + self.stderr.trim() + ); + Ok(()) + } + + pub(crate) fn stdout_is(&self, expected: &str) -> Result { + self.success()?; + crate::ensure!( + self.stdout == expected, + "{} printed {:?}; expected {expected:?}", + self.what, + self.stdout + ); + Ok(()) + } +} diff --git a/tests/testkit/testkit/src/poll.rs b/tests/testkit/testkit/src/poll.rs index 838d89526d..f65592be88 100644 --- a/tests/testkit/testkit/src/poll.rs +++ b/tests/testkit/testkit/src/poll.rs @@ -15,6 +15,18 @@ pub enum Poll { Pending(String), } +/// Fail unless `future` finishes within `timeout`. For a deadline shorter +/// than the step's own, such as one that must fit inside the runner's kill. +pub async fn with_deadline( + what: &str, + timeout: Duration, + future: impl Future>, +) -> Result { + tokio::time::timeout(timeout, future) + .await + .map_err(|_| Error::msg(format!("{what} did not finish within {timeout:?}")))? +} + /// Observe until the condition holds or `timeout` elapses. /// /// An error from `observe` fails immediately. On timeout, the error reports diff --git a/tests/testkit/testkit/src/provider.rs b/tests/testkit/testkit/src/provider.rs index c82901c74a..b68b20989c 100644 --- a/tests/testkit/testkit/src/provider.rs +++ b/tests/testkit/testkit/src/provider.rs @@ -4,7 +4,7 @@ use serde::Deserialize; use crate::context::Context; -use crate::error::{Error, Result, ResultExt as _}; +use crate::error::{Result, ResultExt as _}; use crate::yaml_file::YamlFile; impl Context { @@ -17,7 +17,6 @@ impl Context { id: String, } let id = serde_yml::from_str::(profile) - .map_err(|error| Error::from_error(&error)) .context("read the provider profile's id")? .id; // Deleting a missing profile succeeds, so register the delete before diff --git a/tests/testkit/testkit/src/yaml_file.rs b/tests/testkit/testkit/src/yaml_file.rs index 04ca3d815d..4549c7e95b 100644 --- a/tests/testkit/testkit/src/yaml_file.rs +++ b/tests/testkit/testkit/src/yaml_file.rs @@ -13,10 +13,8 @@ impl YamlFile { let file = tempfile::Builder::new() .suffix(".yaml") .tempfile() - .map_err(|error| Error::from_error(&error)) .context("create a temporary YAML file")?; std::fs::write(file.path(), contents) - .map_err(|error| Error::from_error(&error)) .context(format!("write {}", file.path().display()))?; Ok(Self(file)) }