//! Test commands for running various test suites. //! //! This module provides commands for running different types of tests: //! //! - **Unit tests**: Fast tests that don't execute the brush binary (excludes integration test //! binaries like brush-compat-tests, brush-interactive-tests, brush-completion-tests) //! - **Integration tests**: All workspace tests including unit tests and integration tests that //! execute the brush binary //! - **End-to-end tests**: Real applications' own suites (including bash-completion's), run in //! containers; see `e2e/README.md` //! //! Both unit and integration tests support optional coverage collection via //! `cargo-llvm-cov`. use std::path::{Path, PathBuf}; use anyhow::{Context, Result}; use clap::{Args, Parser, Subcommand}; use xshell::{Shell, cmd}; use crate::common::{BuildProfile, find_target_dir}; /// Integration test binaries that are excluded from unit tests. /// These tests execute the brush binary and are slower. const INTEGRATION_TEST_BINARIES: &[&str] = &[ "brush-compat-tests", "brush-interactive-tests", "brush-completion-tests", ]; #[cfg(windows)] const TEST_BINARIES_DISABLED_ON_WINDOWS: &[&str] = &["brush-compat-tests"]; /// Shared arguments for test commands that need a brush binary. #[derive(Args, Debug, Clone)] pub struct BinaryArgs { /// Path to the brush binary to test. If not specified, uses the binary /// from the workspace's target directory based on --profile/--debug/--release. #[clap(long, global = true)] pub brush_path: Option, /// Build profile to use when auto-detecting the brush binary. #[clap(long, short = 'p', value_enum, default_value_t = BuildProfile::Debug, global = true)] pub profile: BuildProfile, /// Use debug build profile (shorthand for --profile=debug). #[clap(long, conflicts_with_all = ["profile", "release"], global = true)] pub debug: bool, /// Use release build profile (shorthand for --profile=release). #[clap(long, conflicts_with_all = ["profile", "debug"], global = true)] pub release: bool, } impl BinaryArgs { /// Resolve the effective build profile, considering --debug/--release shorthands. #[must_use] pub const fn effective_profile(&self) -> BuildProfile { if self.debug { BuildProfile::Debug } else if self.release { BuildProfile::Release } else { self.profile } } /// Find the brush binary using these arguments. #[cfg(unix)] pub fn find_brush_binary(&self) -> Result { crate::common::find_brush_binary(self.brush_path.as_ref(), self.effective_profile()) } } /// Run tests. #[derive(Parser)] pub struct TestCommand { /// Shared binary arguments. #[clap(flatten)] pub binary_args: BinaryArgs, /// Test subcommand. #[clap(subcommand)] pub subcommand: TestSubcommand, } /// Test subcommands. #[derive(Subcommand, Clone)] pub enum TestSubcommand { /// Run unit tests (fast tests that don't execute the brush binary). /// /// Excludes integration test binaries: brush-compat-tests, brush-interactive-tests, /// brush-completion-tests. Unit(UnitTestArgs), /// Run all workspace tests (unit + integration tests). /// /// This includes all tests: unit tests plus integration tests that execute /// the brush binary (compat tests, interactive tests, completion tests). Integration(IntegrationTestArgs), /// Run containerized end-to-end tests against real applications. #[cfg(unix)] E2e(crate::e2e::E2eArgs), } /// Arguments for unit tests. #[derive(Args, Clone, Default)] pub struct UnitTestArgs { /// Coverage options. #[clap(flatten)] pub coverage: CoverageArgs, } /// Arguments for integration tests. #[derive(Args, Clone, Default)] pub struct IntegrationTestArgs { /// Coverage options. #[clap(flatten)] pub coverage: CoverageArgs, /// Copy the nextest `JUnit` XML results to this path after the test run. /// The copy is performed even if tests fail, so CI can always upload results. #[clap(long)] pub results_output: Option, /// Build and test against a wasm32-wasip3 target under a WASI runtime /// (wasmtime by default). Builds brush for wasm32-wasip3 with its default /// features, then runs a subset of integration tests (excluding compat, /// interactive, and completion suites) under the WASI launcher. Needs a /// toolchain with the wasm32-wasip3 target (Rust 1.100+, or nightly) and /// a runtime with WASI 0.3 support (e.g., wasmtime 46+). #[clap(long)] pub wasi: bool, /// Launcher command for the WASI runtime (only used with --wasi). /// The first token is resolved against `PATH`; subsequent tokens are /// passed as leading arguments before the brush binary path. /// Defaults to `wasmtime run -S inherit-env --dir=.::/ --allow-precompiled --`. #[clap(long)] pub wasi_launcher: Option, /// Skip the WASI wasm build step and assume brush.wasm is already /// present at the expected path (only used with --wasi). #[clap(long)] pub skip_wasi_build: bool, } /// Arguments for coverage collection. #[derive(Args, Clone, Default)] pub struct CoverageArgs { /// Collect code coverage during test run. #[clap(long)] pub coverage: bool, /// Output file for coverage report (Cobertura XML format). /// Only used when --coverage is specified. #[clap(long, short = 'o', default_value = "codecov.xml")] pub coverage_output: PathBuf, } /// Run a test command. pub fn run(cmd: &TestCommand, verbose: bool) -> Result<()> { let sh = Shell::new()?; match &cmd.subcommand { TestSubcommand::Unit(args) => run_unit_tests(&sh, &cmd.binary_args, args, verbose), TestSubcommand::Integration(args) => { run_integration_tests(&sh, &cmd.binary_args, args, verbose) } #[cfg(unix)] TestSubcommand::E2e(args) => crate::e2e::run(&cmd.binary_args, args, verbose), } } /// Run unit tests (excludes integration test binaries). /// /// Unit tests are fast tests that don't execute the brush binary. pub fn run_unit_tests( sh: &Shell, binary_args: &BinaryArgs, args: &UnitTestArgs, verbose: bool, ) -> Result<()> { let profile = binary_args.effective_profile(); eprintln!("Running unit tests ({profile:?} profile)..."); // Build the filter expression to exclude integration test binaries let exclusions: Vec = INTEGRATION_TEST_BINARIES .iter() .map(|name| format!("not binary({name})")) .collect(); let filter_expr = exclusions.join(" and "); if args.coverage.coverage { run_tests_with_coverage( sh, profile, Some(&filter_expr), &args.coverage.coverage_output, verbose, ) } else { run_nextest(sh, profile, Some(&filter_expr), verbose)?; eprintln!("Unit tests passed."); Ok(()) } } /// Run all workspace tests (unit + integration). /// /// This runs all tests in the workspace, including integration tests /// that execute the brush binary. With `--wasi`, builds brush for /// wasm32-wasip3 and runs the integration tests under a WASI runtime. pub fn run_integration_tests( sh: &Shell, binary_args: &BinaryArgs, args: &IntegrationTestArgs, verbose: bool, ) -> Result<()> { let profile = binary_args.effective_profile(); if args.wasi { if args.coverage.coverage { eprintln!("Warning: --coverage is not supported with --wasi and will be ignored."); } return run_integration_tests_wasi(sh, profile, args, verbose); } eprintln!("Running integration tests ({profile:?} profile)..."); #[cfg(windows)] let exclusions: Vec = TEST_BINARIES_DISABLED_ON_WINDOWS .iter() .map(|name| format!("not binary({name})")) .collect(); #[cfg(windows)] let filter_expr = exclusions.join(" and "); #[cfg(windows)] let filter = Some(filter_expr.as_str()); #[cfg(not(windows))] let filter = None; let test_result = if args.coverage.coverage { run_tests_with_coverage(sh, profile, filter, &args.coverage.coverage_output, verbose) } else { run_nextest(sh, profile, filter, verbose).map(|()| { eprintln!("Integration tests passed."); }) }; // Copy nextest results if requested (even on test failure, so CI can upload them). if let Some(ref output) = args.results_output { copy_nextest_results(output)?; } test_result } /// Run the brush integration tests against a wasm32-wasip3 build of brush, /// executed under a WASI runtime. Builds the wasm module first unless /// `--skip-wasi-build` is given, then runs the integration tests via nextest /// with the appropriate environment variables populated for the test harness. fn run_integration_tests_wasi( sh: &Shell, profile: BuildProfile, args: &IntegrationTestArgs, verbose: bool, ) -> Result<()> { let is_release = profile == BuildProfile::Release; // Build the wasm module unless the caller opts out. if !args.skip_wasi_build { eprintln!("Building brush for wasm32-wasip3..."); let mut build_args = vec![ "build", "--target", "wasm32-wasip3", "-p", "brush-shell", "--bin", "brush", ]; if is_release { build_args.push("--release"); } if verbose { eprintln!("Running: cargo {}", build_args.join(" ")); } cmd!(sh, "cargo {build_args...}") .run() .context("failed to build brush for wasm32-wasip3")?; } // Locate the wasm module that was (or should have been) produced. let profile_dir = if is_release { "release" } else { "debug" }; let wasm_path = find_target_dir()? .join("wasm32-wasip3") .join(profile_dir) .join("brush.wasm"); let wasm_path = wasm_path.canonicalize().with_context(|| { format!( "brush.wasm not found at {} — did the build step succeed?", wasm_path.display() ) })?; // The `--dir=.::/` flag maps the host root filesystem into the WASI // sandbox. This is intentionally permissive for testing — tests create // temp dirs and need access to fixtures across the filesystem. This is // NOT a recommended default for production use of brush under WASI. // Pre-compile the wasm module to avoid JIT compilation overhead during // parallel test execution. Without this, multiple concurrent wasmtime // processes each try to JIT-compile brush.wasm simultaneously, which // can exceed test timeouts on CI. eprintln!("Pre-compiling brush.wasm..."); let cwasm_path = wasm_path.with_extension("cwasm"); { let wasm_arg = wasm_path.display().to_string(); let cwasm_arg = cwasm_path.display().to_string(); cmd!(sh, "wasmtime compile {wasm_arg} -o {cwasm_arg}") .run() .context("failed to pre-compile brush.wasm with wasmtime")?; } // The default launcher includes --allow-precompiled so wasmtime accepts // the AOT-compiled .cwasm module without re-compilation, and // `-S inherit-env` so the environment the harness sets for each test // case reaches brush. let launcher = args .wasi_launcher .as_deref() .unwrap_or("wasmtime run -S inherit-env --dir=.::/ --allow-precompiled --"); eprintln!("Running brush integration tests under WASI..."); eprintln!(" wasm: {}", cwasm_path.display()); eprintln!(" launcher: {launcher}"); let brush_path_str = cwasm_path.display().to_string(); let _brush_path = sh.push_env("BRUSH_PATH", &brush_path_str); let _brush_launcher = sh.push_env("BRUSH_LAUNCHER", launcher); let _brush_platform_tags = sh.push_env("BRUSH_PLATFORM_TAGS", "wasi wasm"); // Only run the brush integration tests; compat tests require a native binary. let filter = "binary(brush-integration-tests)"; let test_result = run_nextest(sh, profile, Some(filter), verbose); // Copy nextest results if requested (even on test failure, so CI can upload them). if let Some(ref output) = args.results_output { copy_nextest_results(output)?; } test_result } /// Run cargo nextest with optional filter expression. fn run_nextest( sh: &Shell, profile: BuildProfile, filter_expr: Option<&str>, verbose: bool, ) -> Result<()> { let mut args = vec!["nextest", "run", "--workspace", "--no-fail-fast"]; if profile == BuildProfile::Release { args.push("--release"); } // Add filter expression if provided let filter_value = filter_expr.map(str::to_string); if let Some(ref value) = filter_value { args.push("-E"); args.push(value); } if verbose { eprintln!("Running: cargo {}", args.join(" ")); } cmd!(sh, "cargo {args...}").run().context("Tests failed")?; Ok(()) } /// Copy the nextest `JUnit` XML results to the given output path. fn copy_nextest_results(output: &Path) -> Result<()> { let source = find_target_dir()?.join("nextest/default/test-results.xml"); std::fs::copy(&source, output).with_context(|| { format!( "Failed to copy nextest results from {} to {}", source.display(), output.display() ) })?; eprintln!("Nextest results copied to: {}", output.display()); Ok(()) } /// Run tests with code coverage collection using `cargo-llvm-cov`. /// /// The coverage workflow: /// 1. Source environment variables from `cargo llvm-cov show-env` /// 2. Clean previous coverage data /// 3. Run tests (continuing even if tests fail to still generate report) /// 4. Generate Cobertura XML report for CI integration /// /// Requires `cargo-llvm-cov` to be installed: `cargo install cargo-llvm-cov` fn run_tests_with_coverage( sh: &Shell, profile: BuildProfile, filter_expr: Option<&str>, output: &Path, verbose: bool, ) -> Result<()> { let output_path = output.display().to_string(); eprintln!("Running tests with coverage ({profile:?} profile)..."); eprintln!("Coverage output: {output_path}"); // Set up llvm-cov environment eprintln!("Setting up llvm-cov environment..."); if verbose { eprintln!("Running: cargo llvm-cov show-env --export-prefix"); } let env_output = cmd!(sh, "cargo llvm-cov show-env --export-prefix") .read() .context("Failed to get llvm-cov environment. Is cargo-llvm-cov installed?")?; // Parse and set environment variables from llvm-cov output env_output .lines() .filter_map(|line| line.strip_prefix("export ")) .filter_map(|rest| rest.split_once('=')) .for_each(|(k, v)| sh.set_var(k, v.trim_matches(['"', '\'']))); // Clean previous coverage data if verbose { eprintln!("Running: cargo llvm-cov clean --workspace"); } cmd!(sh, "cargo llvm-cov clean --workspace") .run() .context("Failed to clean coverage data")?; // Build cargo nextest args let mut test_args = vec!["nextest", "run", "--workspace", "--no-fail-fast"]; if profile == BuildProfile::Release { test_args.push("--release"); } // Add filter expression if provided let filter_value = filter_expr.map(str::to_string); if let Some(ref value) = filter_value { test_args.push("-E"); test_args.push(value); } if verbose { eprintln!("Running: cargo {}", test_args.join(" ")); } // Run tests - let output pass through naturally, but continue on failure to generate coverage // report let test_result = cmd!(sh, "cargo {test_args...}").run(); let test_failed = test_result.is_err(); if test_failed { eprintln!("Tests failed, but continuing to generate coverage report..."); } // Generate coverage report (always attempt this) eprintln!("Generating coverage report..."); if verbose { eprintln!("Running: cargo llvm-cov report --cobertura --output-path {output_path}"); } cmd!( sh, "cargo llvm-cov report --cobertura --output-path {output_path}" ) .run() .context("Failed to generate coverage report")?; eprintln!("Coverage report written to: {output_path}"); // Now propagate test failure if tests failed if test_failed { anyhow::bail!("Tests failed (coverage report was still generated)"); } eprintln!("Tests with coverage completed successfully."); Ok(()) }