"""Tests for scripts/install/install.sh, run against real GitHub releases and canary builds on ghcr.io (requires network access). Each test pipes the installer into `sh -s --` (and `bash -s --`), just like `curl ... | sh` would. Failure modes are simulated with shims for curl, gh, tar, and getconf placed ahead of the real tools on PATH. Usage: python3 -m pytest scripts/install/tests (Set GH_TOKEN, e.g. to `$(gh auth token)`, to also verify a real build attestation.) """ import json import os import platform import re import shutil import subprocess import urllib.request from pathlib import Path import pytest INSTALLER = Path(__file__).resolve().parent.parent / "install.sh" # Deliberately not the latest release, so tests can tell pinned installs from latest ones. VERSION = "0.3.0" # Shim name -> (tool it replaces, bash body). $REAL_CURL and $REAL_TAR name the real tools. SHIMS = { "bad-checksum": ( "curl", """ "${REAL_CURL}" "$@" || exit prev="" for arg; do if [[ ${prev} == -o && ${arg} == *.sha256 ]]; then line="$(<"${arg}")" first=0 && [[ ${line} == 0* ]] && first=1 echo "${first}${line:1}" >"${arg}" fi prev="${arg}" done """, ), "bad-attestation": ( "gh", """ [[ $* == *--help ]] && { echo "--source-ref"; exit 0; } echo "simulated attestation failure" >&2 exit 1 """, ), # gh exits 4 when it has no credentials. "gh-unauthenticated": ( "gh", """ [[ $* == *--help ]] && { echo "--source-ref"; exit 0; } echo "To get started with GitHub CLI, please run: gh auth login" >&2 exit 4 """, ), # Accepts every attestation, recording its arguments in $GH_ARGS_FILE. "gh-records-args": ( "gh", """ [[ $* == *--help ]] && { echo "--source-ref"; exit 0; } echo "$*" >"${GH_ARGS_FILE}" """, ), # e.g. the gh shipped by Ubuntu 24.04 and Debian 12: no `attestation` subcommand at all. "gh-too-old": ( "gh", """ echo 'unknown command "attestation" for "gh"' >&2 exit 1 """, ), "broken-binary": ( "tar", """ "${REAL_TAR}" "$@" || exit while [[ $# -gt 0 ]]; do [[ $1 == -C ]] && printf '#!/bin/sh\\nexit 1\\n' >"$2/brush" shift done """, ), "old-glibc": ("getconf", 'echo "glibc 2.17"'), "minimum-glibc": ("getconf", 'echo "glibc 2.34"'), } GH_INSTALLED = shutil.which("gh") is not None # The script is POSIX sh; test it under the system sh (dash on Debian/Ubuntu, busybox ash on # Alpine) and under bash. On macOS, also under the ancient /bin/bash (3.2) if PATH has a newer one. SHELLS = ["sh", "bash"] if platform.system() == "Darwin" and Path(shutil.which("bash")).resolve() != Path("/bin/bash"): SHELLS.append("/bin/bash") @pytest.fixture(params=SHELLS) def install(request, tmp_path): """Runs the installer with the given args, shims, and environment overrides.""" def run(*args, shims=(), **env_overrides): env = os.environ | { "REAL_CURL": shutil.which("curl"), "REAL_TAR": shutil.which("tar"), } for name in shims: tool, body = SHIMS[name] shim = tmp_path / "shims" / name / tool shim.parent.mkdir(parents=True, exist_ok=True) shim.write_text("#!/usr/bin/env bash\n" + body) shim.chmod(0o755) env["PATH"] = f"{shim.parent}{os.pathsep}{env['PATH']}" for key, value in env_overrides.items(): # None unsets the variable if value is None: env.pop(key, None) else: env[key] = str(value) return subprocess.run( [request.param, "-s", "--", *map(str, args)], input=INSTALLER.read_text(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, env=env, ) return run def assert_succeeded(result): assert result.returncode == 0, result.stdout # All output should be the installer's own; anything else means some tool misbehaved. assert all(line.startswith("brush-install: ") for line in result.stdout.splitlines()), ( result.stdout ) def assert_failed(result, message): assert result.returncode != 0, result.stdout # Every failure should end with the installer's own error message. last_line = result.stdout.splitlines()[-1] assert last_line.startswith("brush-install: error: "), result.stdout assert message in last_line, result.stdout def brush_version(path): return subprocess.run( [path, "--version"], capture_output=True, text=True, check=True ).stdout def test_latest_release_to_default_dir(install, tmp_path): home = tmp_path / "home" # Without any other brush on PATH, so the note is about PATH rather than shadowing. path = os.pathsep.join(d for d in os.environ["PATH"].split(os.pathsep) if not (Path(d) / "brush").exists()) result = install(HOME=home, PATH=path, XDG_BIN_HOME=None) assert_succeeded(result) bin_dir = home / ".local" / "bin" version = brush_version(bin_dir / "brush") assert version.startswith("brush ") assert not version.startswith(f"brush {VERSION} ") assert os.listdir(bin_dir) == ["brush"] assert f"note: {bin_dir} is not in your PATH" in result.stdout def test_pinned_version_replaces_existing_binary(install, tmp_path): (tmp_path / "brush").write_text("old") result = install("--version", f"v{VERSION}", "--dir", tmp_path) assert_succeeded(result) assert brush_version(tmp_path / "brush").startswith(f"brush {VERSION} ") def test_xdg_bin_home_is_default_dir(install, tmp_path): xdg_bin = tmp_path / "xdg-bin" result = install("--version", VERSION, HOME=tmp_path / "home", XDG_BIN_HOME=xdg_bin) assert_succeeded(result) assert os.listdir(xdg_bin) == ["brush"] @pytest.mark.skipif(platform.system() != "Linux", reason="glibc detection is Linux-only") def test_old_glibc_falls_back_to_musl(install, tmp_path): result = install("--version", VERSION, "--dir", tmp_path, shims=["old-glibc"]) assert_succeeded(result) assert "-unknown-linux-musl.tar.gz" in result.stdout def host_glibc(): name, version = platform.libc_ver() return tuple(map(int, version.split("."))) if name == "glibc" else None # The shim only steers target selection; the gnu build still has to run on the real libc here. @pytest.mark.skipif(not host_glibc() or host_glibc() < (2, 34), reason="gnu build needs glibc 2.34+") def test_minimum_glibc_selects_gnu_build(install, tmp_path): # Exactly the minimum, so an off-by-one in the version comparison shows up too. result = install("--version", VERSION, "--dir", tmp_path, shims=["minimum-glibc"]) assert_succeeded(result) assert "-unknown-linux-gnu.tar.gz" in result.stdout @pytest.mark.skipif(not GH_INSTALLED, reason="gh is not installed") # CI always provides GH_TOKEN, so never skip there: a missing token should fail, not skip. @pytest.mark.skipif(not os.environ.get("GH_TOKEN") and not os.environ.get("CI"), reason="GH_TOKEN is not set") def test_verifies_real_attestation(install, tmp_path): result = install("--version", VERSION, "--dir", tmp_path, "--require-attestation") assert_succeeded(result) assert "verified GitHub build attestation" in result.stdout # Runs on the CI images that deliberately lack gh (see .github/workflows/install-script.yaml). @pytest.mark.skipif(GH_INSTALLED, reason="gh is installed") def test_missing_gh_warns_with_reason(install, tmp_path): result = install("--version", VERSION, "--dir", tmp_path) assert_succeeded(result) assert "note: GitHub CLI (gh) is not installed, so the build attestation wasn't checked" in result.stdout def test_unauthenticated_gh_warns_with_reason(install, tmp_path): result = install("--version", VERSION, "--dir", tmp_path, shims=["gh-unauthenticated"]) assert_succeeded(result) assert ( "note: GitHub CLI (gh) is not authenticated, so the build attestation wasn't checked" in result.stdout ) def test_too_old_gh_warns_with_reason(install, tmp_path): result = install("--version", VERSION, "--dir", tmp_path, shims=["gh-too-old"]) assert_succeeded(result) assert ( "note: GitHub CLI (gh) is too old (2.68 or newer is needed), so the build attestation wasn't checked" in result.stdout ) def test_notes_when_another_brush_comes_first_in_path(install, tmp_path): earlier, dest = tmp_path / "earlier", tmp_path / "dest" earlier.mkdir() (earlier / "brush").write_text("#!/bin/sh\n") (earlier / "brush").chmod(0o755) path = os.pathsep.join([str(earlier), str(dest), os.environ["PATH"]]) result = install("--version", VERSION, "--dir", dest, PATH=path) assert_succeeded(result) assert f"note: running 'brush' will run {earlier / 'brush'}, not {dest / 'brush'}" in result.stdout def test_no_path_note_when_dir_is_in_path_via_symlink(install, tmp_path): dest, link = tmp_path / "dest", tmp_path / "link" link.symlink_to(dest) path = os.pathsep.join([str(link), os.environ["PATH"]]) result = install("--version", VERSION, "--dir", dest, PATH=path) assert_succeeded(result) # Only the attestation note is allowed (gh may be missing or unauthenticated here). assert not [line for line in result.stdout.splitlines() if "note:" in line and "attestation" not in line] def test_latest_canary(install, tmp_path): result = install("--canary", "--dir", tmp_path) assert_succeeded(result) assert "ghcr.io/reubeno/brush-canary:" in result.stdout assert "+canary " in brush_version(tmp_path / "brush") @pytest.mark.skipif(not GH_INSTALLED, reason="gh is not installed") @pytest.mark.skipif(not os.environ.get("GH_TOKEN") and not os.environ.get("CI"), reason="GH_TOKEN is not set") def test_verifies_real_canary_attestation(install, tmp_path): result = install("--canary", "--dir", tmp_path, "--require-attestation") assert_succeeded(result) assert "verified GitHub build attestation" in result.stdout def newest_canary_commit(): """Returns the commit of the newest canary build, from its manifest on ghcr.io.""" package = "reubeno/brush-canary" with urllib.request.urlopen(f"https://ghcr.io/token?scope=repository:{package}:pull") as response: token = json.load(response)["token"] # Every build publishes all targets, so any target's tag names the same commit. request = urllib.request.Request( f"https://ghcr.io/v2/{package}/manifests/x86_64-unknown-linux-musl", headers={"Authorization": f"Bearer {token}", "Accept": "application/vnd.oci.image.manifest.v1+json"}, ) with urllib.request.urlopen(request) as response: return json.load(response)["annotations"]["org.opencontainers.image.revision"] def test_pinned_canary_attestation_checks_commit(install, tmp_path): commit = newest_canary_commit() args_file = tmp_path / "gh-args" result = install("--commit", commit, "--dir", tmp_path / "bin", shims=["gh-records-args"], GH_ARGS_FILE=args_file) assert_succeeded(result) assert re.search(rf"canary build of commit {commit}, published \d{{4}}-\d\d-\d\dT", result.stdout), result.stdout assert f"--source-digest {commit}" in args_file.read_text() assert f"(git:{commit[:7]}" in brush_version(tmp_path / "bin" / "brush") def test_help(install): result = install("--help", "--bogus") assert result.returncode == 0, result.stdout assert result.stdout.startswith("Installs brush"), result.stdout assert "--require-attestation" in result.stdout def test_unset_home_without_dir(install): assert_failed(install("--version", VERSION, HOME=None, XDG_BIN_HOME=None), "HOME is not set") @pytest.mark.parametrize( "args, message", [ (["--bogus"], "unknown option: --bogus"), (["--version"], "--version requires a value"), (["--dir"], "--dir requires a value"), (["--commit"], "--commit requires a value"), (["--commit", "739a15d"], "--commit needs a full 40-character commit hash"), (["--commit", "0" * 39 + "g"], "--commit needs a full 40-character commit hash"), (["--canary", "--version", VERSION], "--version can't be combined with --canary or --commit"), ], ids=["unknown-option", "missing-version", "missing-dir", "missing-commit", "short-commit", "non-hex-commit", "canary-and-version"], ) def test_bad_arguments(install, args, message): assert_failed(install(*args), message) @pytest.mark.parametrize( "args, shims, message", [ pytest.param( ["--version", "0.0.0"], [], "failed to download", id="nonexistent-version" ), pytest.param( ["--commit", "0" * 40], [], "no canary build of brush found", id="nonexistent-commit" ), pytest.param( ["--version", VERSION], ["bad-checksum"], "checksum mismatch", id="bad-checksum" ), pytest.param( ["--version", VERSION], ["bad-attestation"], "attestation verification failed", id="bad-attestation", ), pytest.param( ["--version", VERSION, "--require-attestation"], ["gh-unauthenticated"], "cannot verify build attestation: GitHub CLI (gh) is not authenticated", id="require-attestation-unauthenticated", ), pytest.param( ["--version", VERSION, "--require-attestation"], ["gh-too-old"], "cannot verify build attestation: GitHub CLI (gh) is too old", id="require-attestation-too-old", ), pytest.param( ["--version", VERSION, "--require-attestation"], [], "cannot verify build attestation: GitHub CLI (gh) is not installed", id="require-attestation-no-gh", # A real gh can't be hidden from PATH, so this only runs where gh is absent. marks=pytest.mark.skipif(GH_INSTALLED, reason="gh is installed"), ), ], ) def test_failure_installs_nothing(install, tmp_path, args, shims, message): dest = tmp_path / "dest" assert_failed(install(*args, "--dir", dest, shims=shims), message) assert not dest.exists() def test_unwritable_dir(install, tmp_path): # A path beneath a regular file can't be created, even by root (as in CI containers). (tmp_path / "file").write_text("") dest = tmp_path / "file" / "bin" assert_failed(install("--version", VERSION, "--dir", dest), f"cannot write to {dest}") def test_broken_binary_keeps_existing_one(install, tmp_path): dest = tmp_path / "dest" dest.mkdir() (dest / "brush").write_text("old") result = install("--version", VERSION, "--dir", dest, shims=["broken-binary"]) assert_failed(result, "build failed to run on this system") assert os.listdir(dest) == ["brush"] assert (dest / "brush").read_text() == "old"