diff --git a/crates/openshell-cli/Cargo.toml b/crates/openshell-cli/Cargo.toml index 8e4cb3fb25..3ce1e5a556 100644 --- a/crates/openshell-cli/Cargo.toml +++ b/crates/openshell-cli/Cargo.toml @@ -86,6 +86,7 @@ workspace = true [features] bundled-z3 = ["openshell-prover/bundled-z3"] +conformance = [] [dev-dependencies] futures = { workspace = true } diff --git a/crates/openshell-cli/src/main.rs b/crates/openshell-cli/src/main.rs index e02c7c9cd5..f84cedd410 100644 --- a/crates/openshell-cli/src/main.rs +++ b/crates/openshell-cli/src/main.rs @@ -554,6 +554,27 @@ enum Commands { command: Option, }, + /// Validate that a gateway and its driver are functioning correctly. + /// + /// Runs a suite of conformance scenarios against the configured gateway + /// and reports pass/fail for each. Exits non-zero if any scenario fails. + /// + /// All sandboxes created by this command use the naming convention + /// `conformance--` and are cleaned up on completion. + /// If the command is interrupted, any remaining sandboxes can be + /// identified by the `conformance-` prefix. + /// + /// Examples: + /// openshell conformance run + /// openshell conformance run --filter lifecycle + /// openshell conformance run --timeout 120 + #[cfg(feature = "conformance")] + #[command(hide = true, help_template = SUBCOMMAND_HELP_TEMPLATE)] + Conformance { + #[command(subcommand)] + command: ConformanceCommands, + }, + // =================================================================== // ADDITIONAL COMMANDS // =================================================================== @@ -1186,6 +1207,47 @@ enum InferenceCommands { }, } +// ----------------------------------------------------------------------- +// Conformance commands +// ----------------------------------------------------------------------- + +#[cfg(feature = "conformance")] +#[derive(Subcommand, Debug)] +enum ConformanceCommands { + /// Run the conformance suite against the configured gateway. + /// + /// Connects to the gateway using the registered credentials and runs + /// each conformance scenario in sequence. Reports pass/fail per + /// scenario and exits non-zero if any scenario fails. + /// + /// Examples: + /// openshell conformance run + /// openshell conformance run --filter lifecycle + /// openshell conformance run --timeout 120 + #[command(help_template = LEAF_HELP_TEMPLATE, next_help_heading = "FLAGS")] + Run { + /// Only run scenarios whose name contains this substring. + #[arg(long, short = 'f')] + filter: Option, + + /// Seconds to wait for each scenario to complete. + #[arg(long, default_value = "300")] + timeout: u64, + + /// Output format. + #[arg(short = 'o', long = "output", value_enum, default_value_t = OutputFormat::Table)] + output: OutputFormat, + }, + + /// List available conformance scenarios without running them. + #[command(help_template = LEAF_HELP_TEMPLATE, next_help_heading = "FLAGS")] + List { + /// Output format. + #[arg(short = 'o', long = "output", value_enum, default_value_t = OutputFormat::Table)] + output: OutputFormat, + }, +} + // ----------------------------------------------------------------------- // Doctor (diagnostic) commands // ----------------------------------------------------------------------- @@ -2062,6 +2124,35 @@ async fn main() -> Result<()> { } }, + // ----------------------------------------------------------- + // Conformance commands + // ----------------------------------------------------------- + #[cfg(feature = "conformance")] + Some(Commands::Conformance { command }) => { + let ctx = resolve_gateway(&cli.gateway, &cli.gateway_endpoint)?; + let mut tls = tls.with_gateway_name(&ctx.name); + apply_auth(&mut tls, &ctx.name); + match command { + ConformanceCommands::Run { + filter, + timeout, + output, + } => { + run::conformance_run( + &ctx.endpoint, + &tls, + filter.as_deref(), + timeout, + output.as_str(), + ) + .await?; + } + ConformanceCommands::List { output } => { + run::conformance_list(output.as_str())?; + } + } + } + // ----------------------------------------------------------- // Doctor (diagnostic) commands // ----------------------------------------------------------- diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index f56bb71512..c552daf441 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -1558,6 +1558,659 @@ pub fn gateway_admin_info(name: &str) -> Result<()> { /// /// Checks Docker connectivity and reports the result. Returns exit code 0 /// if all checks pass, 1 otherwise. +// ----------------------------------------------------------------------- +// Conformance suite +// ----------------------------------------------------------------------- + +/// A single conformance scenario definition. +#[cfg(feature = "conformance")] +struct Scenario { + /// Short lowercase-hyphenated name, used in sandbox naming and filtering. + name: &'static str, + /// Human-readable description shown in `conformance list`. + description: &'static str, +} + +#[cfg(feature = "conformance")] +fn all_scenarios() -> &'static [Scenario] { + &[ + Scenario { + name: "capabilities", + description: "GetCapabilities returns non-empty driver name, version, and default image", + }, + Scenario { + name: "lifecycle", + description: "Create → running → stop → delete completes without error", + }, + Scenario { + name: "not-found", + description: "Get/stop/delete for an unknown sandbox ID returns an appropriate error", + }, + Scenario { + name: "idempotent-delete", + description: "Deleting an already-deleted sandbox does not error", + }, + Scenario { + name: "validate", + description: "Invalid sandbox specs are rejected before creation", + }, + Scenario { + name: "concurrent", + description: "Two sandboxes created simultaneously do not interfere", + }, + Scenario { + name: "labels", + description: "Labels are persisted on create and filter list results correctly", + }, + ] +} + +/// Outcome of a single conformance scenario. +#[cfg(feature = "conformance")] +#[derive(serde::Serialize)] +struct ScenarioResult { + name: String, + passed: bool, + message: String, + duration_ms: u64, +} + +#[cfg(feature = "conformance")] +pub async fn conformance_run( + server: &str, + tls: &TlsOptions, + filter: Option<&str>, + timeout_secs: u64, + output: &str, +) -> Result<()> { + use std::time::Instant; + + let client = grpc_client(server, tls) + .await + .wrap_err("failed to connect to gateway")?; + + // Stable run-id for sandbox naming: seconds since Unix epoch, truncated + // to 8 hex digits. Keeps names Kubernetes RFC 1123 safe and short enough + // to read in `sandbox list` output. + let run_id = format!( + "{:08x}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as u32 + ); + + let scenarios = all_scenarios() + .iter() + .filter(|s| filter.map_or(true, |f| s.name.contains(f))); + + let mut results: Vec = Vec::new(); + let mut any_failed = false; + + for scenario in scenarios { + let start = Instant::now(); + let timeout = Duration::from_secs(timeout_secs); + let mut client = client.clone(); + + let outcome = + tokio::time::timeout(timeout, run_scenario(scenario.name, &mut client, &run_id)) + .await + .unwrap_or_else(|_| { + Err(miette::miette!("scenario timed out after {timeout_secs}s")) + }); + + let passed = outcome.is_ok(); + if !passed { + any_failed = true; + } + + results.push(ScenarioResult { + name: scenario.name.to_string(), + passed, + message: match &outcome { + Ok(()) => "ok".to_string(), + Err(e) => format!("{e}"), + }, + duration_ms: start.elapsed().as_millis() as u64, + }); + } + + if crate::output::print_output_collection(output, &results, |r| { + serde_json::json!({ + "name": r.name, + "passed": r.passed, + "message": r.message, + "duration_ms": r.duration_ms, + }) + })? { + // structured output already printed + } else { + for r in &results { + let status = if r.passed { "PASS" } else { "FAIL" }; + println!( + " [{status}] {} ({}ms) — {}", + r.name, r.duration_ms, r.message + ); + } + } + + if any_failed { + return Err(miette::miette!("one or more conformance scenarios failed")); + } + Ok(()) +} + +/// Dispatch a scenario by name. +#[cfg(feature = "conformance")] +async fn run_scenario(name: &str, client: &mut crate::tls::GrpcClient, run_id: &str) -> Result<()> { + match name { + "lifecycle" => scenario_lifecycle(client, run_id).await, + "not-found" => scenario_not_found(client, run_id).await, + "idempotent-delete" => scenario_idempotent_delete(client, run_id).await, + "validate" => scenario_validate(client).await, + "concurrent" => scenario_concurrent(client, run_id).await, + "labels" => scenario_labels(client, run_id).await, + _ => Err(miette::miette!("scenario '{name}' is not yet implemented")), + } +} + +/// Poll `WatchSandbox` until the sandbox reaches Ready, returning an error on +/// Error phase or a closed stream. Does not perform cleanup — callers are +/// responsible for deleting the sandbox if this returns an error. +#[cfg(feature = "conformance")] +async fn wait_for_ready( + client: &mut crate::tls::GrpcClient, + sandbox_id: &str, + sandbox_name: &str, +) -> Result<()> { + let mut stream = client + .watch_sandbox(WatchSandboxRequest { + id: sandbox_id.to_string(), + follow_status: true, + follow_logs: false, + follow_events: false, + log_tail_lines: 0, + event_tail: 0, + stop_on_terminal: false, + log_since_ms: 0, + log_sources: vec![], + log_min_level: String::new(), + }) + .await + .into_diagnostic() + .wrap_err("watch_sandbox failed")? + .into_inner(); + + while let Some(item) = stream.next().await { + let evt = item + .into_diagnostic() + .wrap_err("watch_sandbox stream error")?; + if let Some(openshell_core::proto::sandbox_stream_event::Payload::Sandbox(s)) = evt.payload + { + match SandboxPhase::try_from(s.phase()).unwrap_or(SandboxPhase::Unknown) { + SandboxPhase::Ready => return Ok(()), + SandboxPhase::Error => { + return Err(miette::miette!( + "sandbox '{sandbox_name}' entered Error phase before becoming Ready" + )); + } + _ => {} + } + } + } + + Err(miette::miette!( + "watch stream ended before sandbox '{sandbox_name}' reached Ready" + )) +} + +/// Scenario: create → ready → delete. +/// +/// Creates a minimal sandbox, waits for it to reach the Ready phase, then +/// deletes it. Verifies that the sandbox appears in the list between create +/// and delete, and that delete reports it as deleted. +#[cfg(feature = "conformance")] +async fn scenario_lifecycle(client: &mut crate::tls::GrpcClient, run_id: &str) -> Result<()> { + let sandbox_name = format!("conformance-lifecycle-{run_id}"); + + // ── 1. Create ──────────────────────────────────────────────────────── + let response = client + .create_sandbox(CreateSandboxRequest { + name: sandbox_name.clone(), + spec: Some(SandboxSpec::default()), + labels: Default::default(), + }) + .await + .into_diagnostic() + .wrap_err("create_sandbox failed")?; + + let sandbox = response + .into_inner() + .sandbox + .ok_or_else(|| miette::miette!("create_sandbox response missing sandbox"))?; + let sandbox_id = sandbox.object_id().to_string(); + + // ── 2. Wait for Ready ──────────────────────────────────────────────── + if let Err(e) = wait_for_ready(client, &sandbox_id, &sandbox_name).await { + let _ = client + .delete_sandbox(DeleteSandboxRequest { + name: sandbox_name.clone(), + }) + .await; + return Err(e); + } + + // ── 3. Verify it appears in the list ──────────────────────────────── + let list_response = client + .list_sandboxes(ListSandboxesRequest::default()) + .await + .into_diagnostic() + .wrap_err("list_sandboxes failed")?; + + let found = list_response + .into_inner() + .sandboxes + .iter() + .any(|s| s.object_name() == sandbox_name); + + if !found { + let _ = client + .delete_sandbox(DeleteSandboxRequest { + name: sandbox_name.clone(), + }) + .await; + return Err(miette::miette!( + "sandbox '{sandbox_name}' not found in list_sandboxes response after creation" + )); + } + + // ── 4. Delete ──────────────────────────────────────────────────────── + let del_response = client + .delete_sandbox(DeleteSandboxRequest { + name: sandbox_name.clone(), + }) + .await + .into_diagnostic() + .wrap_err("delete_sandbox failed")?; + + if !del_response.into_inner().deleted { + return Err(miette::miette!( + "delete_sandbox reported sandbox '{sandbox_name}' was not deleted" + )); + } + + Ok(()) +} + +/// Scenario: get and delete a sandbox that does not exist. +/// +/// Verifies that `GetSandbox` returns `NOT_FOUND` and that `DeleteSandbox` +/// returns `deleted: false` without erroring for a name that was never +/// created. +#[cfg(feature = "conformance")] +async fn scenario_not_found(client: &mut crate::tls::GrpcClient, run_id: &str) -> Result<()> { + let phantom_name = format!("conformance-not-found-{run_id}"); + + // ── 1. GetSandbox → NOT_FOUND ──────────────────────────────────────── + let err = client + .get_sandbox(GetSandboxRequest { + name: phantom_name.clone(), + }) + .await + .expect_err("get_sandbox on a non-existent sandbox should have returned NOT_FOUND"); + + if err.code() != Code::NotFound { + return Err(miette::miette!( + "get_sandbox returned {} instead of NOT_FOUND", + err.code() + )); + } + + // ── 2. DeleteSandbox → ok, deleted: false ──────────────────────────── + let del = client + .delete_sandbox(DeleteSandboxRequest { + name: phantom_name.clone(), + }) + .await + .into_diagnostic() + .wrap_err("delete_sandbox on a non-existent sandbox should not error")? + .into_inner(); + + if del.deleted { + return Err(miette::miette!( + "delete_sandbox reported deleted=true for a sandbox that was never created" + )); + } + + Ok(()) +} + +/// Scenario: delete an already-deleted sandbox does not error. +/// +/// Creates a sandbox, waits for it to be Ready, deletes it (expecting +/// `deleted: true`), then deletes it again and asserts the second call +/// returns `deleted: false` without an error. +#[cfg(feature = "conformance")] +async fn scenario_idempotent_delete( + client: &mut crate::tls::GrpcClient, + run_id: &str, +) -> Result<()> { + let sandbox_name = format!("conformance-idempotent-delete-{run_id}"); + + // ── 1. Create ──────────────────────────────────────────────────────── + let response = client + .create_sandbox(CreateSandboxRequest { + name: sandbox_name.clone(), + spec: Some(SandboxSpec::default()), + labels: Default::default(), + }) + .await + .into_diagnostic() + .wrap_err("create_sandbox failed")?; + + let sandbox = response + .into_inner() + .sandbox + .ok_or_else(|| miette::miette!("create_sandbox response missing sandbox"))?; + let sandbox_id = sandbox.object_id().to_string(); + + // ── 2. Wait for Ready ──────────────────────────────────────────────── + if let Err(e) = wait_for_ready(client, &sandbox_id, &sandbox_name).await { + let _ = client + .delete_sandbox(DeleteSandboxRequest { + name: sandbox_name.clone(), + }) + .await; + return Err(e); + } + + // ── 3. First delete → deleted: true ────────────────────────────────── + let del1 = client + .delete_sandbox(DeleteSandboxRequest { + name: sandbox_name.clone(), + }) + .await + .into_diagnostic() + .wrap_err("first delete_sandbox failed")? + .into_inner(); + + if !del1.deleted { + return Err(miette::miette!( + "first delete_sandbox reported deleted=false for sandbox '{sandbox_name}'" + )); + } + + // ── 4. Second delete → ok, deleted: false ──────────────────────────── + let del2 = client + .delete_sandbox(DeleteSandboxRequest { + name: sandbox_name.clone(), + }) + .await + .into_diagnostic() + .wrap_err("second delete_sandbox (idempotency check) returned an error")? + .into_inner(); + + if del2.deleted { + return Err(miette::miette!( + "second delete_sandbox reported deleted=true — expected deleted=false" + )); + } + + Ok(()) +} + +/// Scenario: invalid sandbox specs are rejected before creation. +/// +/// Verifies that the gateway returns `INVALID_ARGUMENT` for two distinct +/// invalid inputs without creating any sandbox: a missing spec field and a +/// zero GPU count. +#[cfg(feature = "conformance")] +async fn scenario_validate(client: &mut crate::tls::GrpcClient) -> Result<()> { + // ── 1. spec=None → INVALID_ARGUMENT ────────────────────────────────── + let err = client + .create_sandbox(CreateSandboxRequest { + name: String::new(), + spec: None, + labels: Default::default(), + }) + .await + .expect_err("create_sandbox with spec=None should have been rejected"); + + if err.code() != Code::InvalidArgument { + return Err(miette::miette!( + "create_sandbox(spec=None) returned {} instead of INVALID_ARGUMENT", + err.code() + )); + } + + // ── 2. gpu.count=0 → INVALID_ARGUMENT ──────────────────────────────── + let err2 = client + .create_sandbox(CreateSandboxRequest { + name: String::new(), + spec: Some(SandboxSpec { + resource_requirements: Some(ResourceRequirements { + gpu: Some(GpuResourceRequirements { count: Some(0) }), + }), + ..Default::default() + }), + labels: Default::default(), + }) + .await + .expect_err("create_sandbox with gpu.count=0 should have been rejected"); + + if err2.code() != Code::InvalidArgument { + return Err(miette::miette!( + "create_sandbox(gpu.count=0) returned {} instead of INVALID_ARGUMENT", + err2.code() + )); + } + + Ok(()) +} + +/// Scenario: two sandboxes created simultaneously do not interfere. +/// +/// Issues two `CreateSandbox` calls concurrently, waits for both to reach +/// Ready in parallel, verifies both appear in `ListSandboxes`, then deletes +/// both. Any failure cleans up both sandboxes before returning. +#[cfg(feature = "conformance")] +async fn scenario_concurrent(client: &mut crate::tls::GrpcClient, run_id: &str) -> Result<()> { + let name_a = format!("conformance-concurrent-a-{run_id}"); + let name_b = format!("conformance-concurrent-b-{run_id}"); + + // ── 1. Create both sandboxes concurrently ──────────────────────────── + let mut client_a = client.clone(); + let mut client_b = client.clone(); + + let (resp_a, resp_b) = tokio::join!( + client_a.create_sandbox(CreateSandboxRequest { + name: name_a.clone(), + spec: Some(SandboxSpec::default()), + labels: Default::default(), + }), + client_b.create_sandbox(CreateSandboxRequest { + name: name_b.clone(), + spec: Some(SandboxSpec::default()), + labels: Default::default(), + }), + ); + + let sandbox_a = resp_a + .into_diagnostic() + .wrap_err("create_sandbox(a) failed")? + .into_inner() + .sandbox + .ok_or_else(|| miette::miette!("create_sandbox(a) response missing sandbox"))?; + let sandbox_b = resp_b + .into_diagnostic() + .wrap_err("create_sandbox(b) failed")? + .into_inner() + .sandbox + .ok_or_else(|| miette::miette!("create_sandbox(b) response missing sandbox"))?; + + let id_a = sandbox_a.object_id().to_string(); + let id_b = sandbox_b.object_id().to_string(); + + // ── 2. Wait for both to reach Ready concurrently ───────────────────── + let (ready_a, ready_b) = tokio::join!( + wait_for_ready(&mut client_a, &id_a, &name_a), + wait_for_ready(&mut client_b, &id_b, &name_b), + ); + + // Best-effort cleanup before surfacing watch errors. + if ready_a.is_err() || ready_b.is_err() { + let _ = client + .delete_sandbox(DeleteSandboxRequest { + name: name_a.clone(), + }) + .await; + let _ = client + .delete_sandbox(DeleteSandboxRequest { + name: name_b.clone(), + }) + .await; + ready_a?; + ready_b?; + } + + // ── 3. Both appear in the list ─────────────────────────────────────── + let sandboxes = client + .list_sandboxes(ListSandboxesRequest::default()) + .await + .into_diagnostic() + .wrap_err("list_sandboxes failed")? + .into_inner() + .sandboxes; + + let found_a = sandboxes.iter().any(|s| s.object_name() == name_a); + let found_b = sandboxes.iter().any(|s| s.object_name() == name_b); + + // ── 4. Delete both ─────────────────────────────────────────────────── + let _ = client + .delete_sandbox(DeleteSandboxRequest { + name: name_a.clone(), + }) + .await; + let _ = client + .delete_sandbox(DeleteSandboxRequest { + name: name_b.clone(), + }) + .await; + + if !found_a { + return Err(miette::miette!( + "sandbox '{name_a}' not found in list_sandboxes after concurrent create" + )); + } + if !found_b { + return Err(miette::miette!( + "sandbox '{name_b}' not found in list_sandboxes after concurrent create" + )); + } + + Ok(()) +} + +/// Scenario: labels are persisted on create and filter list results correctly. +/// +/// Creates two sandboxes with distinct label values, calls `ListSandboxes` +/// with a label selector, and verifies that only the matching sandbox is +/// returned. Labels are stored by the gateway on creation so no Ready wait +/// is required; both sandboxes are deleted after the assertion. +#[cfg(feature = "conformance")] +async fn scenario_labels(client: &mut crate::tls::GrpcClient, run_id: &str) -> Result<()> { + let name_a = format!("conformance-labels-a-{run_id}"); + let name_b = format!("conformance-labels-b-{run_id}"); + let label_key = "conformance-scenario".to_string(); + + // ── 1. Create two sandboxes with distinct label values ─────────────── + client + .create_sandbox(CreateSandboxRequest { + name: name_a.clone(), + spec: Some(SandboxSpec::default()), + labels: [(label_key.clone(), "labels-a".to_string())] + .into_iter() + .collect(), + }) + .await + .into_diagnostic() + .wrap_err("create_sandbox(a) failed")?; + + client + .create_sandbox(CreateSandboxRequest { + name: name_b.clone(), + spec: Some(SandboxSpec::default()), + labels: [(label_key.clone(), "labels-b".to_string())] + .into_iter() + .collect(), + }) + .await + .into_diagnostic() + .wrap_err("create_sandbox(b) failed")?; + + // ── 2. Filter by label — must return only sandbox A ────────────────── + let filtered = client + .list_sandboxes(ListSandboxesRequest { + label_selector: format!("{label_key}=labels-a"), + ..Default::default() + }) + .await + .into_diagnostic() + .wrap_err("list_sandboxes with label_selector failed")? + .into_inner() + .sandboxes; + + // ── 3. Cleanup ─────────────────────────────────────────────────────── + let _ = client + .delete_sandbox(DeleteSandboxRequest { + name: name_a.clone(), + }) + .await; + let _ = client + .delete_sandbox(DeleteSandboxRequest { + name: name_b.clone(), + }) + .await; + + // ── 4. Assert after cleanup so both sandboxes are always removed ────── + let found_a = filtered.iter().any(|s| s.object_name() == name_a); + let found_b = filtered.iter().any(|s| s.object_name() == name_b); + + if !found_a { + return Err(miette::miette!( + "sandbox '{name_a}' not found in label-filtered list (selector: {label_key}=labels-a)" + )); + } + if found_b { + return Err(miette::miette!( + "sandbox '{name_b}' appeared in label-filtered list but should have been excluded \ + (selector: {label_key}=labels-a)" + )); + } + + Ok(()) +} + +#[cfg(feature = "conformance")] +pub fn conformance_list(output: &str) -> Result<()> { + let scenarios = all_scenarios(); + + if crate::output::print_output_collection(output, scenarios, |s| { + serde_json::json!({ + "name": s.name, + "description": s.description, + }) + })? { + return Ok(()); + } + + println!("{} conformance scenarios:", scenarios.len()); + for s in scenarios { + println!(" {:<20} {}", s.name, s.description); + } + Ok(()) +} + pub fn doctor_check() -> Result<()> { use std::io::Write; let mut stdout = std::io::stdout().lock();