From 7049eec0fa04bb7772d0c28719c84c88559d86a5 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Tue, 2 Jun 2026 12:49:47 -0400 Subject: [PATCH 1/3] test(setup): add experimental end-to-end setup-flow matrix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a non-blocking, data-driven test matrix that verifies the intended `socket-patch setup` flow end to end for every supported ecosystem/package manager: 0. prepare a project with a dependency + a committed patch set 1. run `socket-patch setup` to configure install hooks 2. run the native install command for the package manager 3. check whether the patch was applied (marker on disk) plus negative controls (no setup, empty/wrong-target/alt patch sets). The suite is ASPIRATIONAL and intentionally non-blocking: `setup` only configures npm-family hooks today, so non-npm `baseline_with_setup` cases are expected `known_gap`s — a baseline of what `setup` must eventually support. Results are classified against a recorded baseline; the runner exits non-zero only on a regression. Components: - tests/setup_matrix/matrix.json — declarative cases (targets x scenarios), the single source of truth for both the runner and the Rust wrappers. - tests/setup_matrix/run-case.sh — self-contained bash flow driver (scaffold -> setup -> install -> verify -> JSON); generates the npx/pnpm shims inline so the hook resolves to the local binary instead of fetching the published wrapper. - scripts/setup-matrix.sh — orchestrator (build/run/list/query/results), classifies pass/known_gap/progress/regression, emits machine-readable JSON. - crates/socket-patch-cli/tests/setup_matrix_.rs (+ shared module), gated by a new `setup-e2e` feature; assert the aspirational ideal. - tests/docker/Dockerfile.{npm,pypi} extended additively (pnpm/yarn via corepack; uv/poetry/pdm/hatch) — existing docker_e2e tests unaffected. - ci.yml: a `setup-matrix` job, `continue-on-error: true` (must stay out of required checks). No CLI/source behavior changes. Verified in Docker on all 9 images (socket-patch 3.3.0): 80 cases, 56 pass / 24 known_gap, 0 regression / 0 error; npm/yarn/pnpm/bun apply, everything else is a documented gap; all negative controls pass (no leaks). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 72 ++++ crates/socket-patch-cli/Cargo.toml | 10 + .../tests/setup_matrix_cargo.rs | 14 + .../tests/setup_matrix_common/mod.rs | 284 +++++++++++++ .../tests/setup_matrix_composer.rs | 15 + .../tests/setup_matrix_deno.rs | 16 + .../tests/setup_matrix_gem.rs | 14 + .../tests/setup_matrix_golang.rs | 14 + .../tests/setup_matrix_maven.rs | 15 + .../tests/setup_matrix_npm.rs | 33 ++ .../tests/setup_matrix_nuget.rs | 15 + .../tests/setup_matrix_pypi.rs | 37 ++ scripts/setup-matrix.sh | 275 +++++++++++++ tests/docker/Dockerfile.npm | 14 +- tests/docker/Dockerfile.pypi | 21 +- tests/docker/README.md | 15 + tests/setup_matrix/README.md | 120 ++++++ tests/setup_matrix/matrix.json | 169 ++++++++ tests/setup_matrix/results/.gitignore | 3 + tests/setup_matrix/run-case.sh | 376 ++++++++++++++++++ tests/setup_matrix/shims/npx | 45 +++ tests/setup_matrix/shims/pnpm | 38 ++ 22 files changed, 1606 insertions(+), 9 deletions(-) create mode 100644 crates/socket-patch-cli/tests/setup_matrix_cargo.rs create mode 100644 crates/socket-patch-cli/tests/setup_matrix_common/mod.rs create mode 100644 crates/socket-patch-cli/tests/setup_matrix_composer.rs create mode 100644 crates/socket-patch-cli/tests/setup_matrix_deno.rs create mode 100644 crates/socket-patch-cli/tests/setup_matrix_gem.rs create mode 100644 crates/socket-patch-cli/tests/setup_matrix_golang.rs create mode 100644 crates/socket-patch-cli/tests/setup_matrix_maven.rs create mode 100644 crates/socket-patch-cli/tests/setup_matrix_npm.rs create mode 100644 crates/socket-patch-cli/tests/setup_matrix_nuget.rs create mode 100644 crates/socket-patch-cli/tests/setup_matrix_pypi.rs create mode 100755 scripts/setup-matrix.sh create mode 100644 tests/setup_matrix/README.md create mode 100644 tests/setup_matrix/matrix.json create mode 100644 tests/setup_matrix/results/.gitignore create mode 100755 tests/setup_matrix/run-case.sh create mode 100755 tests/setup_matrix/shims/npx create mode 100755 tests/setup_matrix/shims/pnpm diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bac0a477..02e40a10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -577,3 +577,75 @@ jobs: - name: Run ${{ matrix.ecosystem }} Docker e2e test run: cargo test -p socket-patch-cli --features docker-e2e --test docker_e2e_${{ matrix.ecosystem }} + + # ---------------------------------------------------------------------- + # Experimental `setup`-flow matrix (NON-BLOCKING). + # + # For each ecosystem/package manager, drives the full intended flow — + # prepare deps + a committed patch set, run `socket-patch setup`, run + # the native install, check whether the patch was applied — plus the + # negative controls (no setup, empty/wrong/alt patch sets). See + # tests/setup_matrix/ and scripts/setup-matrix.sh. + # + # This is EXPERIMENTAL and intentionally not required to pass yet: + # `setup` only configures npm-family install hooks today, so most + # non-npm `baseline_with_setup` cases are EXPECTED to fail (a baseline + # of what `setup` must eventually support). `continue-on-error: true` + # means this job never blocks a PR — it must ALSO be left OUT of the + # repo's required status checks (configured in the branch-protection + # UI, not in this file). The orchestrator exits non-zero only on a + # *regression* vs the recorded baseline; the full per-case result set + # is uploaded as a JSON artifact for inspection. + # ---------------------------------------------------------------------- + setup-matrix: + runs-on: ubuntu-latest + continue-on-error: true + permissions: + contents: read + strategy: + fail-fast: false + matrix: + ecosystem: [npm, pypi, cargo, gem, golang, maven, composer, nuget, deno] + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Set up Docker Buildx + # `driver: docker` — the per-ecosystem image's `FROM + # socket-patch-test-base:latest` only resolves when buildx talks + # directly to the host docker daemon (see e2e-docker above). + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + with: + driver: docker + + - name: Install Rust + run: rustup show + + - name: Build base image + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + with: + context: . + file: tests/docker/Dockerfile.base + tags: socket-patch-test-base:latest + load: true + + - name: Build ${{ matrix.ecosystem }} image + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + with: + context: . + file: tests/docker/Dockerfile.${{ matrix.ecosystem }} + tags: socket-patch-test-${{ matrix.ecosystem }}:latest + load: true + + - name: Run ${{ matrix.ecosystem }} setup-matrix + run: scripts/setup-matrix.sh run --ecosystem ${{ matrix.ecosystem }} --out "report-${{ matrix.ecosystem }}.json" + + - name: Upload ${{ matrix.ecosystem }} setup-matrix report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: setup-matrix-${{ matrix.ecosystem }} + path: report-${{ matrix.ecosystem }}.json + if-no-files-found: warn diff --git a/crates/socket-patch-cli/Cargo.toml b/crates/socket-patch-cli/Cargo.toml index 3ce2753d..fba6ef4e 100644 --- a/crates/socket-patch-cli/Cargo.toml +++ b/crates/socket-patch-cli/Cargo.toml @@ -39,6 +39,16 @@ deno = ["socket-patch-core/deno"] # `tests/docker_e2e_*.rs`. Tests in this suite require either a running # Docker daemon OR `SOCKET_PATCH_TEST_HOST=1` (host-toolchain mode). docker-e2e = [] +# Enables the experimental `setup` end-to-end test matrix under +# `tests/setup_matrix_*.rs`, which drives the `socket-patch setup` → +# native-install → patch-applied flow across every ecosystem/package +# manager via `tests/setup_matrix/run-case.sh`. Same runtime requirement +# as docker-e2e (Docker daemon OR `SOCKET_PATCH_TEST_HOST=1`). These +# tests are ASPIRATIONAL: they assert the ideal (install applies the +# patch) and are EXPECTED to fail for ecosystems whose install hooks +# `setup` does not yet configure. Kept off `--all-features`-required CI; +# the dedicated `setup-matrix` CI job runs them non-blocking. +setup-e2e = [] [dev-dependencies] sha2 = { workspace = true } diff --git a/crates/socket-patch-cli/tests/setup_matrix_cargo.rs b/crates/socket-patch-cli/tests/setup_matrix_cargo.rs new file mode 100644 index 00000000..ddea0b8a --- /dev/null +++ b/crates/socket-patch-cli/tests/setup_matrix_cargo.rs @@ -0,0 +1,14 @@ +//! setup-matrix: cargo ecosystem. `setup` is a no-op for Rust projects +//! (no package.json) and cargo has no post-install hook, so the +//! with-setup cases are an EXPECTED BASELINE GAP. +//! +//! Run: `cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_cargo` +#![cfg(feature = "setup-e2e")] + +#[path = "setup_matrix_common/mod.rs"] +mod smc; + +#[test] +fn cargo() { + smc::run_pm("cargo", "cargo"); +} diff --git a/crates/socket-patch-cli/tests/setup_matrix_common/mod.rs b/crates/socket-patch-cli/tests/setup_matrix_common/mod.rs new file mode 100644 index 00000000..4532ab3d --- /dev/null +++ b/crates/socket-patch-cli/tests/setup_matrix_common/mod.rs @@ -0,0 +1,284 @@ +//! Shared harness for the experimental `socket-patch setup` end-to-end +//! test matrix (`tests/setup_matrix_*.rs`, gated by the `setup-e2e` +//! feature). +//! +//! Each `setup_matrix_.rs` wrapper pulls this in with +//! `#[path = "setup_matrix_common/mod.rs"] mod smc;` and calls +//! [`run_pm`] for each package manager it covers. The wrappers are +//! thin; ALL the flow logic lives in the single bash driver +//! `tests/setup_matrix/run-case.sh`, which this module invokes either +//! inside a Docker container (default) or on the host +//! (`SOCKET_PATCH_TEST_HOST=1`). The declarative case list comes from +//! `tests/setup_matrix/matrix.json` — the same spec the +//! `scripts/setup-matrix.sh` orchestrator consumes. +//! +//! ASPIRATIONAL assertion: each case asserts the *ideal* — that after +//! `setup` + a native install, the patch is (or isn't) applied as the +//! scenario expects. For ecosystems whose install hooks `setup` does +//! not yet configure, the `baseline_with_setup` / `alt_content_patchset` +//! cases are EXPECTED to fail; the failure message tags them +//! `BASELINE GAP` so the red is understood as a TODO, not a surprise. +//! +//! `#![allow(dead_code)]` — wrappers use different subsets of this API. + +#![allow(dead_code)] + +use std::path::{Path, PathBuf}; +use std::process::Command; + +/// Path to the built binary under test (host mode passes this to the +/// driver via `SOCKET_PATCH_BIN`). +fn binary() -> PathBuf { + env!("CARGO_BIN_EXE_socket-patch").into() +} + +/// Workspace root = two levels up from this crate's manifest dir. +fn workspace_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(|p| p.parent()) + .expect("workspace root") + .to_path_buf() +} + +fn driver_path() -> PathBuf { + workspace_root().join("tests/setup_matrix/run-case.sh") +} + +fn matrix_path() -> PathBuf { + workspace_root().join("tests/setup_matrix/matrix.json") +} + +/// Host mode runs the driver against host-installed toolchains instead +/// of a container. Mirrors the `docker_e2e_*` convention. +fn host_mode() -> bool { + std::env::var("SOCKET_PATCH_TEST_HOST").map(|v| v == "1").unwrap_or(false) +} + +fn docker_on_path() -> bool { + Command::new("docker") + .arg("--version") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .map(|s| s.success()) + .unwrap_or(false) +} + +fn image_present(image: &str) -> bool { + Command::new("docker") + .args(["image", "inspect", &format!("socket-patch-test-{image}:latest")]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .map(|s| s.success()) + .unwrap_or(false) +} + +/// One concrete case = a (target, scenario) pair from matrix.json. +#[derive(Clone)] +struct Case { + id: String, + ecosystem: String, + pm: String, + image: String, + scenario: String, + patchset: String, + run_setup: bool, + expect_applied: bool, + baseline_supported: bool, + package: String, + version: String, + purl: String, + manifest_key: String, + apply_ecosystems: String, + marker: String, + alt_marker: String, +} + +impl Case { + /// Baseline (currently-known) outcome under today's code: + /// `setup` only wires npm-family hooks, so applied is expected only + /// when the target advertises `baseline_supported` AND the scenario + /// aspires to apply. + fn baseline_applied(&self) -> bool { + self.expect_applied && self.baseline_supported + } + + fn sm_env(&self) -> Vec<(String, String)> { + vec![ + ("SM_ID".into(), self.id.clone()), + ("SM_ECOSYSTEM".into(), self.ecosystem.clone()), + ("SM_PM".into(), self.pm.clone()), + ("SM_SCENARIO".into(), self.scenario.clone()), + ("SM_PATCHSET".into(), self.patchset.clone()), + ("SM_RUN_SETUP".into(), if self.run_setup { "1" } else { "0" }.into()), + ("SM_EXPECT_APPLIED".into(), if self.expect_applied { "1" } else { "0" }.into()), + ("SM_PACKAGE".into(), self.package.clone()), + ("SM_VERSION".into(), self.version.clone()), + ("SM_PURL".into(), self.purl.clone()), + ("SM_MANIFEST_KEY".into(), self.manifest_key.clone()), + ("SM_APPLY_ECOSYSTEMS".into(), self.apply_ecosystems.clone()), + ("SM_MARKER".into(), self.marker.clone()), + ("SM_ALT_MARKER".into(), self.alt_marker.clone()), + ] + } +} + +/// Load every case for a given (ecosystem, pm) by crossing that target +/// with all scenarios in the spec. +fn load_cases(ecosystem: &str, pm: &str) -> Vec { + let text = std::fs::read_to_string(matrix_path()) + .unwrap_or_else(|e| panic!("read matrix.json: {e}")); + let spec: serde_json::Value = + serde_json::from_str(&text).expect("parse matrix.json"); + let marker = spec["marker"].as_str().unwrap_or("").to_string(); + let alt_marker = spec["alt_marker"].as_str().unwrap_or("").to_string(); + + let target = spec["targets"] + .as_array() + .expect("targets array") + .iter() + .find(|t| t["ecosystem"] == ecosystem && t["pm"] == pm) + .unwrap_or_else(|| panic!("no target for {ecosystem}/{pm} in matrix.json")); + + let mut cases = Vec::new(); + for s in spec["scenarios"].as_array().expect("scenarios array") { + let scenario = s["id"].as_str().unwrap().to_string(); + cases.push(Case { + id: format!("{ecosystem}/{pm}/{scenario}"), + ecosystem: ecosystem.to_string(), + pm: pm.to_string(), + image: target["image"].as_str().unwrap().to_string(), + scenario, + patchset: s["patchset"].as_str().unwrap().to_string(), + run_setup: s["run_setup"].as_bool().unwrap(), + expect_applied: s["expect_applied"].as_bool().unwrap(), + baseline_supported: target["baseline_supported"].as_bool().unwrap(), + package: target["package"].as_str().unwrap().to_string(), + version: target["version"].as_str().unwrap().to_string(), + purl: target["purl"].as_str().unwrap().to_string(), + manifest_key: target["manifest_key"].as_str().unwrap().to_string(), + apply_ecosystems: target["apply_ecosystems"].as_str().unwrap().to_string(), + marker: marker.clone(), + alt_marker: alt_marker.clone(), + }); + } + cases +} + +struct RunResult { + actual_applied: bool, + raw: String, + parsed: Option, +} + +/// Execute one case via the bash driver (container or host) and parse +/// its JSON result line. +fn run_case(case: &Case) -> RunResult { + let driver = driver_path(); + let env = case.sm_env(); + + let output = if host_mode() { + let mut cmd = Command::new("bash"); + cmd.arg(&driver); + for (k, v) in &env { + cmd.env(k, v); + } + cmd.env("SOCKET_PATCH_BIN", binary()); + cmd.output().expect("spawn bash driver") + } else { + let script = std::fs::read_to_string(&driver) + .unwrap_or_else(|e| panic!("read driver: {e}")); + let mut cmd = Command::new("docker"); + cmd.args(["run", "--rm"]); + for (k, v) in &env { + cmd.args(["-e", &format!("{k}={v}")]); + } + cmd.arg(format!("socket-patch-test-{}:latest", case.image)); + cmd.args(["bash", "-c", &script]); + cmd.output().expect("spawn docker run") + }; + + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + + // The driver prints its result JSON as the last matching stdout line. + let line = stdout + .lines() + .rev() + .find(|l| l.trim_start().starts_with('{') && l.contains("actual_applied")); + + let parsed = line.and_then(|l| serde_json::from_str::(l).ok()); + let actual_applied = parsed + .as_ref() + .and_then(|v| v["actual_applied"].as_bool()) + .unwrap_or(false); + + RunResult { + actual_applied, + raw: format!("stdout:\n{stdout}\nstderr:\n{stderr}"), + parsed, + } +} + +/// Run every scenario for one (ecosystem, pm) and assert each meets the +/// ASPIRATIONAL expectation. Soft-skips when Docker / the ecosystem +/// image is unavailable (container mode) — matching the `docker_e2e_*` +/// convention where Rust integration tests have no native "skipped". +pub fn run_pm(ecosystem: &str, pm: &str) { + if !host_mode() && !docker_on_path() { + eprintln!("skip {ecosystem}/{pm}: docker not on PATH (set SOCKET_PATCH_TEST_HOST=1 to run on host)"); + return; + } + + let cases = load_cases(ecosystem, pm); + if !host_mode() { + if let Some(c) = cases.first() { + if !image_present(&c.image) { + eprintln!( + "skip {ecosystem}/{pm}: image socket-patch-test-{}:latest not present \ + (build it: scripts/setup-matrix.sh build --ecosystem {})", + c.image, c.image + ); + return; + } + } + } + + let mut failures = Vec::new(); + for case in &cases { + let res = run_case(case); + if res.actual_applied != case.expect_applied { + let tag = if case.baseline_applied() { + // We recorded this as working; failing now is a real regression. + "REGRESSION (baseline says this should apply)" + } else if case.expect_applied { + "BASELINE GAP (setup does not yet wire this package manager)" + } else { + "LEAK (patch applied without the hook configuring it)" + }; + failures.push(format!( + " - {}: expected applied={}, got {} [{}]\n{}", + case.id, case.expect_applied, res.actual_applied, tag, indent(&res.raw) + )); + } + } + + assert!( + failures.is_empty(), + "{}/{}: {} of {} setup-matrix case(s) did not meet the aspirational \ + expectation. BASELINE GAP entries are the experimental TODO list \ + (this suite is non-blocking in CI); REGRESSION / LEAK entries are \ + real problems:\n{}", + ecosystem, + pm, + failures.len(), + cases.len(), + failures.join("\n") + ); +} + +fn indent(s: &str) -> String { + s.lines().map(|l| format!(" {l}")).collect::>().join("\n") +} diff --git a/crates/socket-patch-cli/tests/setup_matrix_composer.rs b/crates/socket-patch-cli/tests/setup_matrix_composer.rs new file mode 100644 index 00000000..8ec68934 --- /dev/null +++ b/crates/socket-patch-cli/tests/setup_matrix_composer.rs @@ -0,0 +1,15 @@ +//! setup-matrix: composer ecosystem (PHP). Composer DOES expose a +//! `post-install-cmd` event hook, but `setup` does not wire it today, +//! so the with-setup cases are an EXPECTED BASELINE GAP — and a clear +//! candidate for the first non-npm ecosystem `setup` could support. +//! +//! Run: `cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_composer` +#![cfg(feature = "setup-e2e")] + +#[path = "setup_matrix_common/mod.rs"] +mod smc; + +#[test] +fn composer() { + smc::run_pm("composer", "composer"); +} diff --git a/crates/socket-patch-cli/tests/setup_matrix_deno.rs b/crates/socket-patch-cli/tests/setup_matrix_deno.rs new file mode 100644 index 00000000..4cec9383 --- /dev/null +++ b/crates/socket-patch-cli/tests/setup_matrix_deno.rs @@ -0,0 +1,16 @@ +//! setup-matrix: deno ecosystem (deno install against a package.json, +//! npm-via-deno layout). `setup` DOES rewrite the package.json (deno +//! projects have one), but whether `deno install` runs the root +//! postinstall hook is uncertain — so the baseline records this as a +//! GAP. If it applies, the orchestrator flags it `progress`. +//! +//! Run: `cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_deno` +#![cfg(feature = "setup-e2e")] + +#[path = "setup_matrix_common/mod.rs"] +mod smc; + +#[test] +fn deno() { + smc::run_pm("deno", "deno"); +} diff --git a/crates/socket-patch-cli/tests/setup_matrix_gem.rs b/crates/socket-patch-cli/tests/setup_matrix_gem.rs new file mode 100644 index 00000000..c5507b54 --- /dev/null +++ b/crates/socket-patch-cli/tests/setup_matrix_gem.rs @@ -0,0 +1,14 @@ +//! setup-matrix: gem ecosystem (bundler). No native post-install hook +//! and `setup` is a no-op, so the with-setup cases are an EXPECTED +//! BASELINE GAP. +//! +//! Run: `cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_gem` +#![cfg(feature = "setup-e2e")] + +#[path = "setup_matrix_common/mod.rs"] +mod smc; + +#[test] +fn bundler() { + smc::run_pm("gem", "bundler"); +} diff --git a/crates/socket-patch-cli/tests/setup_matrix_golang.rs b/crates/socket-patch-cli/tests/setup_matrix_golang.rs new file mode 100644 index 00000000..c444e1d8 --- /dev/null +++ b/crates/socket-patch-cli/tests/setup_matrix_golang.rs @@ -0,0 +1,14 @@ +//! setup-matrix: golang ecosystem (go modules). No native post-install +//! hook and `setup` is a no-op, so the with-setup cases are an EXPECTED +//! BASELINE GAP. +//! +//! Run: `cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_golang` +#![cfg(feature = "setup-e2e")] + +#[path = "setup_matrix_common/mod.rs"] +mod smc; + +#[test] +fn go() { + smc::run_pm("golang", "go"); +} diff --git a/crates/socket-patch-cli/tests/setup_matrix_maven.rs b/crates/socket-patch-cli/tests/setup_matrix_maven.rs new file mode 100644 index 00000000..ab08a169 --- /dev/null +++ b/crates/socket-patch-cli/tests/setup_matrix_maven.rs @@ -0,0 +1,15 @@ +//! setup-matrix: maven ecosystem (mvn). No native post-install hook, +//! `setup` is a no-op, and apply is additionally gated behind +//! `SOCKET_EXPERIMENTAL_MAVEN` (the driver sets it). The with-setup +//! cases are an EXPECTED BASELINE GAP. +//! +//! Run: `cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_maven` +#![cfg(feature = "setup-e2e")] + +#[path = "setup_matrix_common/mod.rs"] +mod smc; + +#[test] +fn mvn() { + smc::run_pm("maven", "mvn"); +} diff --git a/crates/socket-patch-cli/tests/setup_matrix_npm.rs b/crates/socket-patch-cli/tests/setup_matrix_npm.rs new file mode 100644 index 00000000..ac4bd3c0 --- /dev/null +++ b/crates/socket-patch-cli/tests/setup_matrix_npm.rs @@ -0,0 +1,33 @@ +//! setup-matrix: npm ecosystem (npm / yarn / pnpm / bun). +//! +//! These are the ecosystems `socket-patch setup` actually supports +//! today (it writes a package.json postinstall hook), so the +//! `baseline_with_setup` / `alt_content_patchset` cases are expected to +//! PASS here. See `setup_matrix_common/mod.rs` for the harness and +//! `tests/setup_matrix/matrix.json` for the case list. +//! +//! Run: `cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_npm` +#![cfg(feature = "setup-e2e")] + +#[path = "setup_matrix_common/mod.rs"] +mod smc; + +#[test] +fn npm() { + smc::run_pm("npm", "npm"); +} + +#[test] +fn yarn() { + smc::run_pm("npm", "yarn"); +} + +#[test] +fn pnpm() { + smc::run_pm("npm", "pnpm"); +} + +#[test] +fn bun() { + smc::run_pm("npm", "bun"); +} diff --git a/crates/socket-patch-cli/tests/setup_matrix_nuget.rs b/crates/socket-patch-cli/tests/setup_matrix_nuget.rs new file mode 100644 index 00000000..06bf050a --- /dev/null +++ b/crates/socket-patch-cli/tests/setup_matrix_nuget.rs @@ -0,0 +1,15 @@ +//! setup-matrix: nuget ecosystem (dotnet). No native post-install hook, +//! `setup` is a no-op, and apply is additionally gated behind +//! `SOCKET_EXPERIMENTAL_NUGET` (the driver sets it). The with-setup +//! cases are an EXPECTED BASELINE GAP. +//! +//! Run: `cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_nuget` +#![cfg(feature = "setup-e2e")] + +#[path = "setup_matrix_common/mod.rs"] +mod smc; + +#[test] +fn dotnet() { + smc::run_pm("nuget", "dotnet"); +} diff --git a/crates/socket-patch-cli/tests/setup_matrix_pypi.rs b/crates/socket-patch-cli/tests/setup_matrix_pypi.rs new file mode 100644 index 00000000..e11a9210 --- /dev/null +++ b/crates/socket-patch-cli/tests/setup_matrix_pypi.rs @@ -0,0 +1,37 @@ +//! setup-matrix: pypi ecosystem (pip / uv / poetry / pdm / hatch). +//! +//! Python installers have no native post-install hook and `socket-patch +//! setup` is a no-op for them, so the `baseline_with_setup` / +//! `alt_content_patchset` cases are EXPECTED to fail here (BASELINE +//! GAP). The negative-control / empty / wrong-target cases should pass. +//! +//! Run: `cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_pypi` +#![cfg(feature = "setup-e2e")] + +#[path = "setup_matrix_common/mod.rs"] +mod smc; + +#[test] +fn pip() { + smc::run_pm("pypi", "pip"); +} + +#[test] +fn uv() { + smc::run_pm("pypi", "uv"); +} + +#[test] +fn poetry() { + smc::run_pm("pypi", "poetry"); +} + +#[test] +fn pdm() { + smc::run_pm("pypi", "pdm"); +} + +#[test] +fn hatch() { + smc::run_pm("pypi", "hatch"); +} diff --git a/scripts/setup-matrix.sh b/scripts/setup-matrix.sh new file mode 100755 index 00000000..67faa4ba --- /dev/null +++ b/scripts/setup-matrix.sh @@ -0,0 +1,275 @@ +#!/usr/bin/env bash +# ===================================================================== +# setup-matrix.sh — orchestrate and query the `socket-patch setup` +# end-to-end test matrix. +# +# The matrix asks, for every supported ecosystem/package-manager: +# "does `socket-patch setup` configure things so that a normal install +# applies the project's patches?" Each case runs the flow driver +# (tests/setup_matrix/run-case.sh) which prepares a project + committed +# patch set, optionally runs `socket-patch setup`, runs the native +# install, and checks whether the patch landed on disk. +# +# Results are classified against the recorded baseline in matrix.json: +# pass meets the ideal AND matches the recorded baseline +# known_gap fails the ideal but exactly as recorded (expected today) +# progress better than the recorded baseline (update baseline!) +# regression diverged from the baseline the wrong way (this is the +# only thing that makes `run` exit non-zero) +# error the driver could not produce a result +# +# Subcommands: +# build [--ecosystem E]... build base + per-ecosystem images +# run [--ecosystem E] [--pm P] [--scenario S] [--host] [--out FILE] [--verbose] +# list [--json] enumerate every matrix case +# query [--status S] [--ecosystem E] [--pm P] [--scenario S] filter latest results +# results print the latest aggregate +# +# CLI/agent-friendly: `list`/`query`/`results` emit JSON; `run` writes a +# machine-readable report to tests/setup_matrix/results/latest.json. +# ===================================================================== +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SM_DIR="$REPO_ROOT/tests/setup_matrix" +MATRIX="$SM_DIR/matrix.json" +DRIVER="$SM_DIR/run-case.sh" +RESULTS_DIR="$SM_DIR/results" +LATEST="$RESULTS_DIR/latest.json" + +ALL_ECOSYSTEMS=(npm pypi cargo gem golang maven composer nuget deno) + +die() { echo "error: $*" >&2; exit 1; } +need() { command -v "$1" >/dev/null 2>&1 || die "'$1' is required but not on PATH"; } + +usage() { sed -n '2,40p' "$0" | sed 's/^# \{0,1\}//'; } + +need jq +[ -f "$MATRIX" ] || die "matrix spec not found: $MATRIX" + +# Emit one TSV row per case (target x scenario), honoring filters. +# Columns: id eco pm image hook_family baseline_supported package version +# purl manifest_key apply_ecosystems scenario patchset run_setup +# expect_applied +cases_tsv() { # $1=eco-filter ("" = all) $2=pm-filter $3=scenario-filter + jq -r --arg eco "${1:-}" --arg pm "${2:-}" --arg scn "${3:-}" ' + .marker as $m | .alt_marker as $am | + .targets[] as $t | .scenarios[] as $s | + select($eco == "" or $t.ecosystem == $eco) | + select($pm == "" or $t.pm == $pm) | + select($scn == "" or $s.id == $scn) | + [ ($t.ecosystem + "/" + $t.pm + "/" + $s.id), + $t.ecosystem, $t.pm, $t.image, $t.hook_family, + ($t.baseline_supported|tostring), + $t.package, $t.version, $t.purl, $t.manifest_key, $t.apply_ecosystems, + $s.id, $s.patchset, ($s.run_setup|tostring), ($s.expect_applied|tostring) + ] | @tsv + ' "$MATRIX" +} + +marker() { jq -r '.marker' "$MATRIX"; } +alt_marker() { jq -r '.alt_marker' "$MATRIX"; } + +# --------------------------------------------------------------------- build +cmd_build() { + local ecos=(); + while [ $# -gt 0 ]; do case "$1" in + --ecosystem) ecos+=("$2"); shift 2;; + *) die "build: unknown arg '$1'";; + esac; done + [ ${#ecos[@]} -eq 0 ] && ecos=("${ALL_ECOSYSTEMS[@]}") + need docker + echo ">> building base image" >&2 + docker build -f "$REPO_ROOT/tests/docker/Dockerfile.base" -t socket-patch-test-base:latest "$REPO_ROOT" \ + || die "base image build failed" + local e + for e in "${ecos[@]}"; do + echo ">> building $e image" >&2 + docker build -f "$REPO_ROOT/tests/docker/Dockerfile.$e" -t "socket-patch-test-$e:latest" "$REPO_ROOT" \ + || die "$e image build failed" + done + echo ">> done" >&2 +} + +# --------------------------------------------------------------------- list +cmd_list() { + local as_json=0 + while [ $# -gt 0 ]; do case "$1" in --json) as_json=1; shift;; *) die "list: unknown arg '$1'";; esac; done + if [ "$as_json" = 1 ]; then + jq '[ .targets[] as $t | .scenarios[] as $s | + { id: ($t.ecosystem+"/"+$t.pm+"/"+$s.id), ecosystem:$t.ecosystem, pm:$t.pm, + scenario:$s.id, image:$t.image, hook_family:$t.hook_family, + baseline_supported:$t.baseline_supported, expect_applied:$s.expect_applied } ]' "$MATRIX" + else + printf '%-44s %-9s %-8s %-22s %s\n' ID ECO PM SCENARIO EXPECT + cases_tsv "" "" "" | while IFS=$'\t' read -r id eco pm image hook bsup pkg ver purl key aeco scn pset rsetup expect; do + printf '%-44s %-9s %-8s %-22s %s\n' "$id" "$eco" "$pm" "$scn" "$expect" + done + fi +} + +# --------------------------------------------------------------------- run +resolve_host_bin() { + if [ -n "${SOCKET_PATCH_BIN:-}" ]; then echo "$SOCKET_PATCH_BIN"; return; fi + for c in "$REPO_ROOT/target/release/socket-patch" "$REPO_ROOT/target/debug/socket-patch"; do + [ -x "$c" ] && { echo "$c"; return; } + done + command -v socket-patch 2>/dev/null || echo "" +} + +cmd_run() { + local eco="" pm="" scn="" host=0 out="$LATEST" verbose=0 + while [ $# -gt 0 ]; do case "$1" in + --ecosystem) eco="$2"; shift 2;; + --pm) pm="$2"; shift 2;; + --scenario) scn="$2"; shift 2;; + --host) host=1; shift;; + --out) out="$2"; shift 2;; + --verbose) verbose=1; shift;; + *) die "run: unknown arg '$1'";; + esac; done + + local MARK ALT; MARK="$(marker)"; ALT="$(alt_marker)" + mkdir -p "$RESULTS_DIR" + local jsonl; jsonl="$(mktemp)" + + if [ "$host" = 0 ]; then need docker; fi + local host_bin="" + if [ "$host" = 1 ]; then + host_bin="$(resolve_host_bin)" + [ -n "$host_bin" ] || die "host mode: no socket-patch binary found (build it or set SOCKET_PATCH_BIN)" + echo ">> host mode, binary: $host_bin" >&2 + fi + + local total=0 + while IFS=$'\t' read -r id eco_ pm_ image hook bsup pkg ver purl key aeco scn_ pset rsetup expect; do + [ -z "$id" ] && continue + total=$((total+1)) + echo ">> [$total] $id" >&2 + + # Common SM_* env for the driver. + local -a base_env=( + "SM_ID=$id" "SM_ECOSYSTEM=$eco_" "SM_PM=$pm_" "SM_SCENARIO=$scn_" + "SM_PATCHSET=$pset" "SM_RUN_SETUP=$([ "$rsetup" = true ] && echo 1 || echo 0)" + "SM_EXPECT_APPLIED=$([ "$expect" = true ] && echo 1 || echo 0)" + "SM_PACKAGE=$pkg" "SM_VERSION=$ver" "SM_PURL=$purl" + "SM_MANIFEST_KEY=$key" "SM_APPLY_ECOSYSTEMS=$aeco" + "SM_MARKER=$MARK" "SM_ALT_MARKER=$ALT" + ) + + local raw="" rc=0 + if [ "$host" = 1 ]; then + if [ "$verbose" = 1 ]; then + raw="$(env "${base_env[@]}" "SOCKET_PATCH_BIN=$host_bin" bash "$DRIVER")"; rc=$? + else + raw="$(env "${base_env[@]}" "SOCKET_PATCH_BIN=$host_bin" bash "$DRIVER" 2>/dev/null)"; rc=$? + fi + else + local -a docker_env=() + local kv; for kv in "${base_env[@]}"; do docker_env+=(-e "$kv"); done + if [ "$verbose" = 1 ]; then + raw="$(docker run --rm "${docker_env[@]}" "socket-patch-test-$image:latest" bash -c "$(cat "$DRIVER")")"; rc=$? + else + raw="$(docker run --rm "${docker_env[@]}" "socket-patch-test-$image:latest" bash -c "$(cat "$DRIVER")" 2>/dev/null)"; rc=$? + fi + fi + + # The driver prints the result JSON as the last line of stdout. + local result; result="$(printf '%s\n' "$raw" | grep -E '^\{.*"actual_applied"' | tail -n1)" + + # baseline_applied = expect_applied AND baseline_supported. + local bl=false + if [ "$expect" = true ] && [ "$bsup" = true ]; then bl=true; fi + + if [ -n "$result" ] && printf '%s' "$result" | jq -e . >/dev/null 2>&1; then + printf '%s\n' "$result" | jq -c --argjson bl "$bl" --arg img "$image" --arg hk "$hook" ' + . as $r | + ($r.actual_applied == $r.expect_applied) as $ideal | + ($r.actual_applied == $bl) as $base | + (if $ideal and $base then "pass" + elif $ideal and ($base|not) then "progress" + elif ($ideal|not) and $base then "known_gap" + else "regression" end) as $cls | + $r + {baseline_applied:$bl, classification:$cls, image:$img, hook_family:$hk, driver_rc:'"$rc"'} + ' >> "$jsonl" + else + # No parseable result — surface as an error case. + jq -nc --arg id "$id" --arg eco "$eco_" --arg pm "$pm_" --arg scn "$scn_" \ + --arg pset "$pset" --arg img "$image" --arg hk "$hook" --argjson bl "$bl" ' + { id:$id, ecosystem:$eco, pm:$pm, scenario:$scn, patchset:$pset, + expect_applied:null, actual_applied:null, baseline_applied:$bl, + classification:"error", image:$img, hook_family:$hk, driver_rc:'"$rc"', + notes:"driver produced no parseable result" }' >> "$jsonl" + fi + done < <(cases_tsv "$eco" "$pm" "$scn") + + # Aggregate + summarize. + jq -s --arg generated "$(date -u +%FT%TZ)" ' + { generated:$generated, + summary: ( reduce .[] as $c ( + {total:0,pass:0,known_gap:0,progress:0,regression:0,error:0}; + .total += 1 | .[$c.classification] += 1 ) ), + cases: . }' "$jsonl" > "$out" + rm -f "$jsonl" + [ "$out" != "$LATEST" ] && cp "$out" "$LATEST" + + print_summary "$out" + local regressions; regressions="$(jq -r '.summary.regression' "$out")" + if [ "$regressions" -gt 0 ]; then + echo "!! $regressions regression(s) — a case that should work no longer does" >&2 + return 1 + fi + return 0 +} + +print_summary() { # $1 = results file + local f="$1" + echo "" >&2 + printf '%-44s %-8s %-6s %-6s %s\n' CASE PM APPLIED EXPECT STATUS >&2 + jq -r '.cases[] | [ .id, .pm, (.actual_applied|tostring), (.expect_applied|tostring), .classification ] | @tsv' "$f" \ + | while IFS=$'\t' read -r id pm act exp cls; do + printf '%-44s %-8s %-6s %-6s %s\n' "$id" "$pm" "$act" "$exp" "$cls" >&2 + done + echo "" >&2 + jq -r '.summary | "total=\(.total) pass=\(.pass) known_gap=\(.known_gap) progress=\(.progress) regression=\(.regression) error=\(.error)"' "$f" >&2 + local prog; prog="$(jq -r '.summary.progress' "$f")" + [ "$prog" -gt 0 ] && echo ">> $prog case(s) now BETTER than baseline — consider updating baseline_supported in matrix.json" >&2 + echo ">> full report: $f" >&2 +} + +# --------------------------------------------------------------------- query / results +cmd_query() { + local status="" eco="" pm="" scn="" + while [ $# -gt 0 ]; do case "$1" in + --status) status="$2"; shift 2;; + --ecosystem) eco="$2"; shift 2;; + --pm) pm="$2"; shift 2;; + --scenario) scn="$2"; shift 2;; + *) die "query: unknown arg '$1'";; + esac; done + [ -f "$LATEST" ] || die "no results yet — run '$0 run' first" + jq --arg st "$status" --arg eco "$eco" --arg pm "$pm" --arg scn "$scn" ' + [ .cases[] + | select($st == "" or .classification == $st) + | select($eco == "" or .ecosystem == $eco) + | select($pm == "" or .pm == $pm) + | select($scn == "" or .scenario == $scn) ]' "$LATEST" +} + +cmd_results() { + [ -f "$LATEST" ] || die "no results yet — run '$0 run' first" + cat "$LATEST" +} + +# --------------------------------------------------------------------- dispatch +[ $# -ge 1 ] || { usage; exit 1; } +sub="$1"; shift || true +case "$sub" in + build) cmd_build "$@";; + run) cmd_run "$@";; + list) cmd_list "$@";; + query) cmd_query "$@";; + results) cmd_results "$@";; + -h|--help|help) usage;; + *) die "unknown subcommand '$sub' (try: build run list query results)";; +esac diff --git a/tests/docker/Dockerfile.npm b/tests/docker/Dockerfile.npm index 31b3d418..9ed0a413 100644 --- a/tests/docker/Dockerfile.npm +++ b/tests/docker/Dockerfile.npm @@ -21,6 +21,18 @@ RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ RUN curl -fsSL https://bun.sh/install | bash \ && ln -s /root/.bun/bin/bun /usr/local/bin/bun +# Enable pnpm and yarn via corepack (bundled with Node 20). The +# setup-matrix suite (tests/setup_matrix/) drives all four npm-family +# package managers — npm, yarn, pnpm, bun — through the `socket-patch +# setup` install-hook flow. `corepack prepare … --activate` pins and +# activates a known version so the binaries resolve without a network +# round-trip at test time. Additive: the existing docker_e2e_npm tests +# are unaffected. +RUN corepack enable \ + && corepack prepare pnpm@9.15.0 --activate \ + && corepack prepare yarn@1.22.22 --activate + # Verify versions are sane at image-build time so a broken NodeSource setup # fails the image build rather than every downstream test. -RUN node --version && npm --version && bun --version && socket-patch --version +RUN node --version && npm --version && bun --version \ + && pnpm --version && yarn --version && socket-patch --version diff --git a/tests/docker/Dockerfile.pypi b/tests/docker/Dockerfile.pypi index 8e7ea3ed..8cab8df2 100644 --- a/tests/docker/Dockerfile.pypi +++ b/tests/docker/Dockerfile.pypi @@ -1,14 +1,16 @@ -# pypi ecosystem test image: base + Python 3.11 + pip + venv + uv. +# pypi ecosystem test image: base + Python 3.11 + pip + venv + uv + +# poetry + pdm + hatch. # # Debian 12 ships Python 3.11. We use a venv inside each test to keep # pip from needing `--break-system-packages` and to match real-world # user flow. # -# uv is installed from PyPI (single self-contained wheel) so the same -# image can drive both the pip-based and uv-based e2e tests. The -# `--break-system-packages` flag is what Debian-packaged pip3 requires -# to install into the system site-packages; it's safe inside the -# disposable test container. +# uv/poetry/pdm/hatch are installed from PyPI so the one image can drive +# every Python package manager the setup-matrix suite exercises +# (tests/setup_matrix/). The `--break-system-packages` flag is what +# Debian-packaged pip3 requires to install into the system site-packages; +# it's safe inside the disposable test container. Additive: the existing +# docker_e2e_pypi tests (pip + uv) are unaffected. FROM socket-patch-test-base:latest RUN apt-get update \ @@ -17,7 +19,10 @@ RUN apt-get update \ python3-pip \ python3-venv \ && rm -rf /var/lib/apt/lists/* \ - && pip3 install --break-system-packages --no-cache-dir uv \ + && pip3 install --break-system-packages --no-cache-dir uv poetry pdm hatch \ && python3 --version \ && pip3 --version \ - && uv --version + && uv --version \ + && poetry --version \ + && pdm --version \ + && hatch --version diff --git a/tests/docker/README.md b/tests/docker/README.md index 8089d0f6..c4b128bc 100644 --- a/tests/docker/README.md +++ b/tests/docker/README.md @@ -108,3 +108,18 @@ Fixtures are synthetic. Real Socket patches are not required to exist for the tested PURLs — what's validated is that the crawler discovers real installed packages and the CLI dispatches correctly through the ecosystem. + +## Related: the `setup`-flow matrix + +A separate, **experimental** suite lives under `tests/setup_matrix/` and +reuses these same per-ecosystem images. Where `docker_e2e_*` drives +`scan → apply` explicitly, the setup-matrix instead runs `socket-patch +setup` and then a *native install* to check whether the configured +install hook applies the patch on its own — the thing `setup` is meant +to enable. It also adds the npm-family package managers (pnpm/yarn via +corepack) and the Python ones (uv/poetry/pdm/hatch), which is why +`Dockerfile.npm` and `Dockerfile.pypi` install those tools. See +`tests/setup_matrix/README.md` for details and the +`scripts/setup-matrix.sh` runner. That suite's CI job (`setup-matrix`) +is **non-blocking** (`continue-on-error: true`) and is expected to fail +for ecosystems whose hooks `setup` does not yet configure. diff --git a/tests/setup_matrix/README.md b/tests/setup_matrix/README.md new file mode 100644 index 00000000..911503cd --- /dev/null +++ b/tests/setup_matrix/README.md @@ -0,0 +1,120 @@ +# `setup`-flow test matrix (experimental) + +This suite verifies the **intended** end-to-end behavior of +`socket-patch setup`: that after `setup` configures a project, a normal +package-manager install applies the project's patches *on its own*, with +no explicit `scan`/`apply` step. + +It is **experimental and non-blocking**. `setup` only configures +npm-family install hooks today, so most non-npm cases are *expected to +fail*. The suite encodes the **aspirational** end state and records a +per-case **baseline** of what works now — the failing cases are a TODO +list for `setup`, not a broken test. + +## The flow (per case) + +Every case runs the same four steps via the bash driver `run-case.sh`: + +0. **prepare** a throwaway project: declare the dependency and commit a + patch set (`.socket/manifest.json` + `.socket/blobs/`). +1. **`socket-patch setup`** — configure install hooks (skipped in the + `no_setup_control` scenario). +2. **native install** — `npm install` / `pip install` / `cargo fetch` / + … for the package manager under test. +3. **check** — is the patch's marker now on disk in the installed file? + +The apply step is fully offline (`SOCKET_OFFLINE=1 SOCKET_FORCE=1`, +inherited by the hook), so the only network use is the real package +install. No Socket API is contacted. + +## Dimensions + +`ecosystem × package-manager × scenario` — see `matrix.json` (the single +source of truth, consumed by both the runner script and the Rust +wrappers). + +- **Package managers:** npm, yarn, pnpm, bun · pip, uv, poetry, pdm, + hatch · cargo · bundler · go · mvn · composer · dotnet · deno. +- **Scenarios:** + - `baseline_with_setup` — setup + install ⇒ patch applied *(ideal)*. + - `no_setup_control` — install only ⇒ NOT applied *(the hook is the cause)*. + - `empty_patchset` — empty manifest ⇒ NOT applied. + - `wrong_target_patchset` — manifest targets a different package ⇒ NOT applied. + - `alt_content_patchset` — a second patch set ⇒ its marker applied *(content tracks the manifest)*. + +## Result classification + +Each case's `actual` is compared against both the aspirational `expect` +and the recorded `baseline`: + +| classification | meaning | +|---|---| +| `pass` | meets the ideal and matches the baseline | +| `known_gap` | fails the ideal, exactly as recorded — expected today, non-blocking | +| `progress` | better than the recorded baseline — update `baseline_supported` in `matrix.json`! | +| `regression` | diverged from the baseline the wrong way — the only thing that fails the runner | +| `error` | the driver produced no parseable result | + +The Rust wrappers (`tests/setup_matrix_.rs`) assert the **ideal** +(`actual == expect`), so they are red for `known_gap` cases — that is the +intended "TODO list" view. The `scripts/setup-matrix.sh` runner uses the +**baseline** view and only exits non-zero on a `regression`. + +## Running it + +Requires a Docker daemon (default) or host-installed toolchains +(`SOCKET_PATCH_TEST_HOST=1`). + +```sh +# Build the shared base + a per-ecosystem image. +scripts/setup-matrix.sh build --ecosystem npm + +# Run all npm-family cases and write a JSON report. +scripts/setup-matrix.sh run --ecosystem npm + +# Filter to a single package manager / scenario. +scripts/setup-matrix.sh run --ecosystem pypi --pm uv +scripts/setup-matrix.sh run --scenario no_setup_control + +# Query the last results (agent-friendly JSON). +scripts/setup-matrix.sh query --status known_gap +scripts/setup-matrix.sh query --status regression +scripts/setup-matrix.sh list --json + +# Host mode (no Docker; needs the toolchains + a built binary on PATH). +SOCKET_PATCH_TEST_HOST=1 scripts/setup-matrix.sh run --ecosystem npm --host +``` + +Or via `cargo test` (the aspirational view; gated by the `setup-e2e` +feature; soft-skips when the image isn't built): + +```sh +cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_npm +SOCKET_PATCH_TEST_HOST=1 cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_npm +``` + +## Files + +- `matrix.json` — declarative case list (targets × scenarios) + markers. +- `run-case.sh` — self-contained flow driver (one case → JSON result); + generates the runner shims inline so it can be piped into a container. +- `shims/{npx,pnpm}` — reference copies of the PATH shims that route + `npx`/`pnpm dlx @socketsecurity/socket-patch` to the locally-built + binary (so the hook runs the binary under test, not a registry fetch). +- `results/latest.json` — most recent aggregate report (git-ignored). +- `../docker/Dockerfile.{npm,pypi,…}` — the per-ecosystem images + (npm/pypi extended with the extra package managers). +- `../../crates/socket-patch-cli/tests/setup_matrix_.rs` — thin Rust + wrappers around the same driver. + +## Adding a package manager / ecosystem + +1. Add a `targets[]` entry to `matrix.json` (image, package, purl, + manifest key, whether `setup` supports it today via + `baseline_supported`). +2. Teach `run-case.sh` how to scaffold + install + resolve the target + file for the new `pm` (the `scaffold_project` / `run_install` / + `resolve_target` case statements). +3. If a new toolchain is needed, add it to the relevant + `tests/docker/Dockerfile.`. +4. Add a `#[test]` for the `pm` in the matching `setup_matrix_.rs`. diff --git a/tests/setup_matrix/matrix.json b/tests/setup_matrix/matrix.json new file mode 100644 index 00000000..65a1a177 --- /dev/null +++ b/tests/setup_matrix/matrix.json @@ -0,0 +1,169 @@ +{ + "_comment": [ + "Declarative source of truth for the `socket-patch setup` end-to-end test matrix.", + "Consumed by BOTH scripts/setup-matrix.sh (jq) and the Rust wrappers", + "crates/socket-patch-cli/tests/setup_matrix_.rs (serde_json).", + "", + "A 'case' is the cross-product (target x scenario). expect_applied comes from", + "the scenario (the ASPIRATIONAL ideal); baseline_supported on the target says", + "whether `setup` ACTUALLY wires a working install hook today. The classifier in", + "the orchestrator compares actual vs both: meeting the ideal => pass; failing the", + "ideal but matching the recorded baseline => known_gap (non-blocking); diverging", + "from the baseline in the wrong direction => regression (blocking the optional job).", + "", + "Packages, PURLs, manifest keys and install layouts are reused verbatim from the", + "existing tests/docker_e2e_.rs so the fixtures are known-valid.", + "NOTE: pypi uses NO `package/` prefix in the manifest key (the python crawler", + "reports the site-packages root); every other ecosystem uses `package/`." + ], + + "marker": "SOCKET-PATCH-SETUP-MATRIX-MARKER", + "alt_marker": "SOCKET-PATCH-SETUP-MATRIX-ALT-MARKER", + + "scenarios": [ + { + "id": "baseline_with_setup", + "run_setup": true, + "patchset": "primary", + "expect_applied": true, + "description": "Prepare deps + committed patch set, run `socket-patch setup`, run the native install. The install hook should apply the patch (the ideal)." + }, + { + "id": "no_setup_control", + "run_setup": false, + "patchset": "primary", + "expect_applied": false, + "description": "Negative control: identical fixture but setup is NOT run. With no hook configured, the install must NOT apply the patch." + }, + { + "id": "empty_patchset", + "run_setup": true, + "patchset": "empty", + "expect_applied": false, + "description": "Different patch set: an empty manifest. Even with setup, nothing should be applied." + }, + { + "id": "wrong_target_patchset", + "run_setup": true, + "patchset": "wrong", + "expect_applied": false, + "description": "Different patch set: a manifest that patches a different, non-installed package. The installed package must be left untouched." + }, + { + "id": "alt_content_patchset", + "run_setup": true, + "patchset": "alt", + "expect_applied": true, + "description": "Different patch set: a second fixture whose blob carries the ALT marker. Proves the applied bytes track the active manifest (alt marker present, primary marker absent)." + } + ], + + "targets": [ + { + "ecosystem": "npm", "pm": "npm", "image": "npm", "hook_family": "npm", + "baseline_supported": true, + "package": "minimist", "version": "1.2.2", "purl": "pkg:npm/minimist@1.2.2", + "manifest_key": "package/index.js", "apply_ecosystems": "npm" + }, + { + "ecosystem": "npm", "pm": "yarn", "image": "npm", "hook_family": "npm", + "baseline_supported": true, + "package": "minimist", "version": "1.2.2", "purl": "pkg:npm/minimist@1.2.2", + "manifest_key": "package/index.js", "apply_ecosystems": "npm" + }, + { + "ecosystem": "npm", "pm": "pnpm", "image": "npm", "hook_family": "pnpm", + "baseline_supported": true, + "package": "minimist", "version": "1.2.2", "purl": "pkg:npm/minimist@1.2.2", + "manifest_key": "package/index.js", "apply_ecosystems": "npm" + }, + { + "ecosystem": "npm", "pm": "bun", "image": "npm", "hook_family": "npm", + "baseline_supported": true, + "package": "minimist", "version": "1.2.2", "purl": "pkg:npm/minimist@1.2.2", + "manifest_key": "package/index.js", "apply_ecosystems": "npm" + }, + + { + "ecosystem": "pypi", "pm": "pip", "image": "pypi", "hook_family": "none", + "baseline_supported": false, + "package": "six", "version": "1.16.0", "purl": "pkg:pypi/six@1.16.0", + "manifest_key": "six.py", "apply_ecosystems": "pypi" + }, + { + "ecosystem": "pypi", "pm": "uv", "image": "pypi", "hook_family": "none", + "baseline_supported": false, + "package": "six", "version": "1.16.0", "purl": "pkg:pypi/six@1.16.0", + "manifest_key": "six.py", "apply_ecosystems": "pypi" + }, + { + "ecosystem": "pypi", "pm": "poetry", "image": "pypi", "hook_family": "none", + "baseline_supported": false, + "package": "six", "version": "1.16.0", "purl": "pkg:pypi/six@1.16.0", + "manifest_key": "six.py", "apply_ecosystems": "pypi" + }, + { + "ecosystem": "pypi", "pm": "pdm", "image": "pypi", "hook_family": "none", + "baseline_supported": false, + "package": "six", "version": "1.16.0", "purl": "pkg:pypi/six@1.16.0", + "manifest_key": "six.py", "apply_ecosystems": "pypi" + }, + { + "ecosystem": "pypi", "pm": "hatch", "image": "pypi", "hook_family": "none", + "baseline_supported": false, + "package": "six", "version": "1.16.0", "purl": "pkg:pypi/six@1.16.0", + "manifest_key": "six.py", "apply_ecosystems": "pypi" + }, + + { + "ecosystem": "cargo", "pm": "cargo", "image": "cargo", "hook_family": "none", + "baseline_supported": false, + "package": "cfg-if", "version": "1.0.0", "purl": "pkg:cargo/cfg-if@1.0.0", + "manifest_key": "package/src/lib.rs", "apply_ecosystems": "cargo" + }, + + { + "ecosystem": "gem", "pm": "bundler", "image": "gem", "hook_family": "none", + "baseline_supported": false, + "package": "colorize", "version": "1.1.0", "purl": "pkg:gem/colorize@1.1.0", + "manifest_key": "package/lib/colorize.rb", "apply_ecosystems": "gem" + }, + + { + "ecosystem": "golang", "pm": "go", "image": "golang", "hook_family": "none", + "baseline_supported": false, + "package": "github.com/gin-gonic/gin", "version": "v1.9.1", + "purl": "pkg:golang/github.com/gin-gonic/gin@v1.9.1", + "manifest_key": "package/gin.go", "apply_ecosystems": "golang" + }, + + { + "ecosystem": "maven", "pm": "mvn", "image": "maven", "hook_family": "none", + "baseline_supported": false, + "package": "org.apache.commons:commons-lang3", "version": "3.12.0", + "purl": "pkg:maven/org.apache.commons/commons-lang3@3.12.0", + "manifest_key": "package/commons-lang3-3.12.0.pom", "apply_ecosystems": "maven" + }, + + { + "ecosystem": "composer", "pm": "composer", "image": "composer", "hook_family": "composer-event", + "baseline_supported": false, + "package": "monolog/monolog", "version": "3.5.0", "purl": "pkg:composer/monolog/monolog@3.5.0", + "manifest_key": "package/src/Monolog/Logger.php", "apply_ecosystems": "composer" + }, + + { + "ecosystem": "nuget", "pm": "dotnet", "image": "nuget", "hook_family": "none", + "baseline_supported": false, + "package": "Newtonsoft.Json", "version": "13.0.3", "purl": "pkg:nuget/newtonsoft.json@13.0.3", + "manifest_key": "package/LICENSE.md", "apply_ecosystems": "nuget" + }, + + { + "ecosystem": "deno", "pm": "deno", "image": "deno", "hook_family": "npm-via-deno", + "baseline_supported": false, + "package": "minimist", "version": "1.2.2", "purl": "pkg:npm/minimist@1.2.2", + "manifest_key": "package/index.js", "apply_ecosystems": "npm" + } + ] +} diff --git a/tests/setup_matrix/results/.gitignore b/tests/setup_matrix/results/.gitignore new file mode 100644 index 00000000..d5ae1810 --- /dev/null +++ b/tests/setup_matrix/results/.gitignore @@ -0,0 +1,3 @@ +# Generated setup-matrix run reports; keep the directory, ignore contents. +* +!.gitignore diff --git a/tests/setup_matrix/run-case.sh b/tests/setup_matrix/run-case.sh new file mode 100755 index 00000000..668db381 --- /dev/null +++ b/tests/setup_matrix/run-case.sh @@ -0,0 +1,376 @@ +#!/usr/bin/env bash +# ===================================================================== +# setup-matrix flow driver — runs ONE (ecosystem, pm, scenario) case of +# the `socket-patch setup` end-to-end matrix and emits a JSON result. +# +# This script is the single source of truth for the flow: +# 0. prepare a project with the dependency + a committed patch set +# 1. (optionally) run `socket-patch setup` to configure install hooks +# 2. run the native install command for the package manager +# 3. check whether the patch was applied (marker present on disk) +# +# It is invoked by BOTH scripts/setup-matrix.sh (orchestrator) and the +# Rust wrappers (crates/socket-patch-cli/tests/setup_matrix_.rs), +# either inside a Docker container (script piped to `bash -c`) or on the +# host. It is fully self-contained: it generates the npx/pnpm shims +# inline so no extra files need to be copied into the container. +# +# The driver only REPORTS (expected vs actual). Pass/fail/known-gap/ +# regression classification is done by the caller against the recorded +# baseline in matrix.json. +# +# Inputs (environment, all SM_*-prefixed): +# SM_ID stable case id (for the JSON result) +# SM_ECOSYSTEM npm|pypi|cargo|gem|golang|maven|composer|nuget|deno +# SM_PM npm|yarn|pnpm|bun|pip|uv|poetry|pdm|hatch|cargo| +# bundler|go|mvn|composer|dotnet|deno +# SM_SCENARIO scenario id (echoed back) +# SM_PATCHSET primary|alt|empty|wrong +# SM_RUN_SETUP 1|0 — run `socket-patch setup` before install +# SM_EXPECT_APPLIED 1|0 — the aspirational expectation +# SM_PACKAGE dependency name (e.g. minimist, six, cfg-if) +# SM_VERSION dependency version (e.g. 1.2.2) +# SM_PURL manifest key PURL (e.g. pkg:npm/minimist@1.2.2) +# SM_MANIFEST_KEY file key in the patch record (e.g. package/index.js, +# or `six.py` for pypi — NO package/ prefix) +# SM_APPLY_ECOSYSTEMS ecosystem token used to build the "wrong" PURL +# SM_MARKER primary marker string spliced into the patched blob +# SM_ALT_MARKER alternate marker (alt_content_patchset) +# SOCKET_PATCH_BIN path to the binary under test (default: socket-patch on PATH) +# SM_WORKDIR scratch dir (default: a fresh mktemp -d) +# ===================================================================== + +set -uo pipefail + +# Route all ordinary output to stderr; the final JSON goes to the saved +# stdout (fd 3) so the result line is the ONLY thing on real stdout. +exec 3>&1 1>&2 + +SM_ID="${SM_ID:-unknown}" +SM_ECOSYSTEM="${SM_ECOSYSTEM:-}" +SM_PM="${SM_PM:-}" +SM_SCENARIO="${SM_SCENARIO:-}" +SM_PATCHSET="${SM_PATCHSET:-primary}" +SM_RUN_SETUP="${SM_RUN_SETUP:-1}" +SM_EXPECT_APPLIED="${SM_EXPECT_APPLIED:-0}" +SM_PACKAGE="${SM_PACKAGE:-}" +SM_VERSION="${SM_VERSION:-}" +SM_PURL="${SM_PURL:-}" +SM_MANIFEST_KEY="${SM_MANIFEST_KEY:-package/index.js}" +SM_APPLY_ECOSYSTEMS="${SM_APPLY_ECOSYSTEMS:-npm}" +SM_MARKER="${SM_MARKER:-SOCKET-PATCH-SETUP-MATRIX-MARKER}" +SM_ALT_MARKER="${SM_ALT_MARKER:-SOCKET-PATCH-SETUP-MATRIX-ALT-MARKER}" + +ZEROHASH="0000000000000000000000000000000000000000000000000000000000000000" +UUID="aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" +WRONG_PURL="pkg:${SM_APPLY_ECOSYSTEMS}/sm-setup-matrix-absent@9.9.9" + +SP_BIN="${SOCKET_PATCH_BIN:-$(command -v socket-patch 2>/dev/null || echo socket-patch)}" +export SOCKET_PATCH_BIN="$SP_BIN" + +NOTES="" +note() { NOTES="${NOTES}${NOTES:+; }$*"; } +log() { printf '[setup-matrix:%s] %s\n' "$SM_ID" "$*"; } + +# --- JSON emit (hand-rolled; values are simple, sanitized) ------------ +json_str() { printf '%s' "$1" | tr -d '\r' | tr '\n' ' ' | sed 's/\\/\\\\/g; s/"/\\"/g'; } +emit_result() { + local actual="$1" primary_present="$2" setup_exit="$3" install_exit="$4" target="$5" status="$6" + printf '{"id":"%s","ecosystem":"%s","pm":"%s","scenario":"%s","patchset":"%s","run_setup":%s,"expect_applied":%s,"actual_applied":%s,"primary_marker_present":%s,"setup_exit":%s,"install_exit":%s,"target":"%s","status":"%s","notes":"%s"}\n' \ + "$(json_str "$SM_ID")" "$(json_str "$SM_ECOSYSTEM")" "$(json_str "$SM_PM")" \ + "$(json_str "$SM_SCENARIO")" "$(json_str "$SM_PATCHSET")" \ + "$([ "$SM_RUN_SETUP" = 1 ] && echo true || echo false)" \ + "$([ "$SM_EXPECT_APPLIED" = 1 ] && echo true || echo false)" \ + "$actual" "$primary_present" "$setup_exit" "$install_exit" \ + "$(json_str "$target")" "$(json_str "$status")" "$(json_str "$NOTES")" >&3 +} + +# --- git-sha256 (blob \0 + content) ------------------------------ +git_sha256() { # $1 = file + local len; len="$(wc -c < "$1")" + { printf 'blob %d\0' "$len"; cat "$1"; } | sha256sum | cut -d' ' -f1 +} + +# --- inline npx/pnpm shims (kept in sync with tests/setup_matrix/shims/) -- +write_shims() { # $1 = shim dir + local d="$1"; mkdir -p "$d" + cat > "$d/npx" <<'SHIM' +#!/usr/bin/env bash +set -uo pipefail +sp_bin="${SOCKET_PATCH_BIN:-socket-patch}" +shim_dir="${SETUP_MATRIX_SHIM_DIR:-}" +clean_path="$PATH" +[ -n "$shim_dir" ] && clean_path="$(printf '%s' "$PATH" | tr ':' '\n' | grep -vxF "$shim_dir" | paste -sd: -)" +real_npx="$(PATH="$clean_path" command -v npx 2>/dev/null || true)" +i=0 +for arg in "$@"; do + case "$arg" in + @socketsecurity/socket-patch|@socketsecurity/socket-patch@*|@socketsecurity/socket-patch/*) + shift "$((i + 1))"; exec "$sp_bin" "$@" ;; + esac + i=$((i + 1)) +done +[ -n "$real_npx" ] && exec "$real_npx" "$@" +echo "setup-matrix npx shim: real npx not found: $*" >&2; exit 127 +SHIM + cat > "$d/pnpm" <<'SHIM' +#!/usr/bin/env bash +set -uo pipefail +sp_bin="${SOCKET_PATCH_BIN:-socket-patch}" +shim_dir="${SETUP_MATRIX_SHIM_DIR:-}" +clean_path="$PATH" +[ -n "$shim_dir" ] && clean_path="$(printf '%s' "$PATH" | tr ':' '\n' | grep -vxF "$shim_dir" | paste -sd: -)" +real_pnpm="$(PATH="$clean_path" command -v pnpm 2>/dev/null || true)" +if [ "${1:-}" = "dlx" ] || [ "${1:-}" = "exec" ]; then + case "${2:-}" in + @socketsecurity/socket-patch|@socketsecurity/socket-patch@*) shift 2; exec "$sp_bin" "$@" ;; + esac +fi +[ -n "$real_pnpm" ] && exec "$real_pnpm" "$@" +echo "setup-matrix pnpm shim: real pnpm not found: $*" >&2; exit 127 +SHIM + chmod +x "$d/npx" "$d/pnpm" +} + +# --- committed patch fixture ------------------------------------------ +write_manifest() { # $1=purl $2=key $3=afterHash + cat > .socket/manifest.json < .socket/manifest.json + note "empty manifest" ;; + wrong) + # A patch for a package that is NOT installed: nothing should match. + local body="/* $SM_MARKER */"; printf '%s\n' "$body" > /tmp/sm_blob + local h; h="$(git_sha256 /tmp/sm_blob)"; cp /tmp/sm_blob ".socket/blobs/$h" + write_manifest "$WRONG_PURL" "$SM_MANIFEST_KEY" "$h" + note "manifest targets absent purl $WRONG_PURL" ;; + alt) + local body="/* $SM_ALT_MARKER */"; printf '%s\n' "$body" > /tmp/sm_blob + local h; h="$(git_sha256 /tmp/sm_blob)"; cp /tmp/sm_blob ".socket/blobs/$h" + write_manifest "$SM_PURL" "$SM_MANIFEST_KEY" "$h" ;; + *) # primary + local body="/* $SM_MARKER */"; printf '%s\n' "$body" > /tmp/sm_blob + local h; h="$(git_sha256 /tmp/sm_blob)"; cp /tmp/sm_blob ".socket/blobs/$h" + write_manifest "$SM_PURL" "$SM_MANIFEST_KEY" "$h" ;; + esac +} + +# --- per-PM project scaffold (must exist before setup runs) ----------- +scaffold_project() { + case "$SM_PM" in + npm|yarn|bun) + printf '{"name":"sm-proj","version":"0.0.0","private":true}\n' > package.json ;; + pnpm) + # pnpm only runs the ROOT postinstall on `pnpm install` (not on + # `pnpm add`), so the dependency is declared up front and installed + # via a bare `pnpm install`. The stub lockfile is the pnpm marker + # that makes `setup` detect pnpm and write the `pnpm dlx` hook. + cat > package.json < pnpm-lock.yaml ;; + deno) + cat > package.json < deno.json < pyproject.toml <"] +package-mode = false + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" +EOF + ;; + pdm) + cat > pyproject.toml < pyproject.toml < Cargo.toml < src/main.rs ;; + bundler) + cat > Gemfile < go.mod ;; + mvn|composer|dotnet) : ;; + esac +} + +# --- per-PM native install (the hook, if configured, fires here) ------ +run_install() { + case "$SM_PM" in + npm) npm install --silent --no-audit --no-fund "$SM_PACKAGE@$SM_VERSION" ;; + yarn) yarn add --silent "$SM_PACKAGE@$SM_VERSION" ;; + pnpm) pnpm install --no-frozen-lockfile ;; + bun) bun add "$SM_PACKAGE@$SM_VERSION" ;; + deno) deno install --allow-scripts ;; + pip) python3 -m venv venv && ./venv/bin/pip install --disable-pip-version-check --quiet --no-cache-dir "$SM_PACKAGE==$SM_VERSION" ;; + uv) uv venv venv && uv pip install --python venv/bin/python --quiet "$SM_PACKAGE==$SM_VERSION" ;; + poetry) poetry config virtualenvs.in-project true --local && poetry add --no-interaction "$SM_PACKAGE@$SM_VERSION" ;; + pdm) pdm config python.use_venv true >/dev/null 2>&1; pdm add "$SM_PACKAGE==$SM_VERSION" ;; + hatch) HATCH_DATA_DIR="$PWD/.hatch" hatch env create && HATCH_DATA_DIR="$PWD/.hatch" hatch run python -c "import ${SM_PACKAGE//-/_}" ;; + cargo) cargo fetch ;; + bundler) bundle config set --local path vendor/bundle && bundle install ;; + go) GOFLAGS=-mod=mod go mod download "$SM_PACKAGE@$SM_VERSION" ;; + mvn) mvn -q -B dependency:get -Dartifact="$SM_PACKAGE:$SM_VERSION" ;; + composer) composer require --quiet --no-interaction "$SM_PACKAGE:$SM_VERSION" ;; + dotnet) dotnet new classlib -o . --force >/dev/null 2>&1 && dotnet add package "$SM_PACKAGE" --version "$SM_VERSION" ;; + *) echo "unknown pm: $SM_PM"; return 2 ;; + esac +} + +# --- resolve the on-disk file the patch would land in ----------------- +resolve_target() { + local rel="${SM_MANIFEST_KEY#package/}" + local base; base="$(basename "$rel")" + case "$SM_ECOSYSTEM" in + npm|deno) printf '%s\n' "$PWD/node_modules/$SM_PACKAGE/$rel" ;; + pypi) find "$PWD" -name "$base" 2>/dev/null | head -1 ;; + cargo) find "${CARGO_HOME:-$HOME/.cargo}/registry/src" -path "*/${SM_PACKAGE}-${SM_VERSION}/${rel}" 2>/dev/null | head -1 ;; + gem) find "$PWD/vendor" -path "*/${SM_PACKAGE}-${SM_VERSION}/${rel}" 2>/dev/null | head -1 ;; + golang) local gmc; gmc="$(go env GOMODCACHE 2>/dev/null || echo "${GOPATH:-$HOME/go}/pkg/mod")"; find "$gmc" -path "*/$(basename "$SM_PACKAGE")@${SM_VERSION}/${rel}" 2>/dev/null | head -1 ;; + maven) find "$HOME/.m2/repository" -name "$base" 2>/dev/null | head -1 ;; + composer) printf '%s\n' "$PWD/vendor/${SM_PACKAGE}/${rel}" ;; + nuget) local lc; lc="$(printf '%s' "$SM_PACKAGE" | tr '[:upper:]' '[:lower:]')"; find "${NUGET_PACKAGES:-$HOME/.nuget/packages}" -path "*/${lc}/${SM_VERSION}/${rel}" 2>/dev/null | head -1 ;; + esac +} + +# ============================ main ==================================== +log "binary: $SP_BIN ($("$SP_BIN" --version 2>/dev/null || echo '??'))" + +WORKDIR="${SM_WORKDIR:-$(mktemp -d)}" +PROJ="$WORKDIR/proj" +mkdir -p "$PROJ" +cd "$PROJ" || { emit_result false null null null "" fail; exit 0; } +note "proj=$PROJ" + +# 0. dependencies + committed patch set +scaffold_project +build_fixture + +# npm-family (incl. deno-via-npm) need the runner shim so the hook's +# `npx`/`pnpm dlx @socketsecurity/socket-patch` resolves to $SP_BIN. +case "$SM_PM" in + npm|yarn|pnpm|bun|deno) + SHIM_DIR="$PROJ/.sp-shims" + write_shims "$SHIM_DIR" + export SETUP_MATRIX_SHIM_DIR="$SHIM_DIR" + export PATH="$SHIM_DIR:$PATH" + log "shims installed at $SHIM_DIR (PATH prepended)" ;; +esac + +# Hermetic apply env inherited by the install hook's `socket-patch apply`. +# NOTE: SOCKET_OFFLINE/SOCKET_FORCE must be "true"/"false" — the apply +# `--force` flag (unlike its siblings) has no boolish value parser, so +# SOCKET_FORCE=1 is rejected with "invalid value '1' for '--force'". +# The SOCKET_EXPERIMENTAL_* gates are read directly from the env and use +# "1". +export SOCKET_OFFLINE=true SOCKET_FORCE=true SOCKET_API_TOKEN=fake SOCKET_ORG_SLUG=test-org +export SOCKET_TELEMETRY_DISABLED=1 SOCKET_EXPERIMENTAL_MAVEN=1 SOCKET_EXPERIMENTAL_NUGET=1 +export SOCKET_CWD="$PROJ" + +# 1. setup (configures hooks; no-op where there is no package.json) +SETUP_EXIT="null" +if [ "$SM_RUN_SETUP" = 1 ]; then + log "running: socket-patch setup --yes" + "$SP_BIN" setup --yes --json; SETUP_EXIT=$? + log "setup exit=$SETUP_EXIT" + [ -f package.json ] && { log "package.json scripts after setup:"; grep -A6 '"scripts"' package.json || true; } +fi + +# 2. native install (this is where a configured hook fires) +log "running install for pm=$SM_PM" +run_install; INSTALL_EXIT=$? +log "install exit=$INSTALL_EXIT" + +# 3. verify +TARGET="$(resolve_target || true)" +log "resolved target: ${TARGET:-}" +APPLIED=false +PRIMARY_PRESENT=null +if [ -n "$TARGET" ] && [ -f "$TARGET" ]; then + # The marker we expect depends on the patch set. + check_marker="$SM_MARKER" + [ "$SM_PATCHSET" = alt ] && check_marker="$SM_ALT_MARKER" + if grep -q "$check_marker" "$TARGET" 2>/dev/null; then APPLIED=true; fi + if grep -q "$SM_MARKER" "$TARGET" 2>/dev/null; then PRIMARY_PRESENT=true; else PRIMARY_PRESENT=false; fi + log "marker '$check_marker' present: $APPLIED" +else + note "target file not found" +fi + +# Driver-level status: did actual match the aspirational expectation? +want=$([ "$SM_EXPECT_APPLIED" = 1 ] && echo true || echo false) +STATUS=fail +[ "$APPLIED" = "$want" ] && STATUS=pass + +emit_result "$APPLIED" "$PRIMARY_PRESENT" "$SETUP_EXIT" "$INSTALL_EXIT" "${TARGET:-}" "$STATUS" +exit 0 diff --git a/tests/setup_matrix/shims/npx b/tests/setup_matrix/shims/npx new file mode 100755 index 00000000..fb158c29 --- /dev/null +++ b/tests/setup_matrix/shims/npx @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# setup-matrix test shim for `npx`. +# +# The hook that `socket-patch setup` writes into package.json is +# npx @socketsecurity/socket-patch apply --silent --ecosystems npm +# In a hermetic test we do NOT want `npx` to fetch the published wrapper +# package from the npm registry — we want it to run the locally-built +# binary under test. This shim, prepended to PATH, intercepts exactly that +# invocation and execs the local binary; every other `npx` call is +# delegated to the real `npx`. +# +# This is a TEST FIXTURE, not a change to socket-patch behavior. It is the +# standalone/reference copy; tests/setup_matrix/run-case.sh embeds an +# identical copy inline so the driver is self-contained when piped into a +# container. Keep the two in sync. +set -uo pipefail + +sp_bin="${SOCKET_PATCH_BIN:-socket-patch}" +shim_dir="${SETUP_MATRIX_SHIM_DIR:-}" + +# Resolve the real npx by searching PATH with our own shim dir removed. +clean_path="$PATH" +if [ -n "$shim_dir" ]; then + clean_path="$(printf '%s' "$PATH" | tr ':' '\n' | grep -vxF "$shim_dir" | paste -sd: -)" +fi +real_npx="$(PATH="$clean_path" command -v npx 2>/dev/null || true)" + +# If any argument names our package, drop everything up to and including it +# and exec the local binary with the remaining apply args. +i=0 +for arg in "$@"; do + case "$arg" in + @socketsecurity/socket-patch|@socketsecurity/socket-patch@*|@socketsecurity/socket-patch/*) + shift "$((i + 1))" + exec "$sp_bin" "$@" + ;; + esac + i=$((i + 1)) +done + +if [ -n "$real_npx" ]; then + exec "$real_npx" "$@" +fi +echo "setup-matrix npx shim: real npx not found and args are not our package: $*" >&2 +exit 127 diff --git a/tests/setup_matrix/shims/pnpm b/tests/setup_matrix/shims/pnpm new file mode 100755 index 00000000..7f342f71 --- /dev/null +++ b/tests/setup_matrix/shims/pnpm @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# setup-matrix test shim for `pnpm`. +# +# For pnpm projects `socket-patch setup` writes the hook +# pnpm dlx @socketsecurity/socket-patch apply --silent --ecosystems npm +# `pnpm dlx` always downloads from the registry, so it cannot be satisfied +# from a local file: dependency. This shim, prepended to PATH, intercepts +# `pnpm dlx @socketsecurity/socket-patch …` (and `pnpm exec …`) and execs +# the locally-built binary; every other `pnpm` invocation — crucially the +# real `pnpm install` / `pnpm add` — is delegated unchanged to the real +# pnpm. +# +# TEST FIXTURE only. Reference copy; run-case.sh embeds an identical copy. +set -uo pipefail + +sp_bin="${SOCKET_PATCH_BIN:-socket-patch}" +shim_dir="${SETUP_MATRIX_SHIM_DIR:-}" + +clean_path="$PATH" +if [ -n "$shim_dir" ]; then + clean_path="$(printf '%s' "$PATH" | tr ':' '\n' | grep -vxF "$shim_dir" | paste -sd: -)" +fi +real_pnpm="$(PATH="$clean_path" command -v pnpm 2>/dev/null || true)" + +if [ "${1:-}" = "dlx" ] || [ "${1:-}" = "exec" ]; then + case "${2:-}" in + @socketsecurity/socket-patch|@socketsecurity/socket-patch@*) + shift 2 + exec "$sp_bin" "$@" + ;; + esac +fi + +if [ -n "$real_pnpm" ]; then + exec "$real_pnpm" "$@" +fi +echo "setup-matrix pnpm shim: real pnpm not found: $*" >&2 +exit 127 From 145f2db31bceedfbb7decb7a223aa553d592eac4 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Tue, 2 Jun 2026 13:11:34 -0400 Subject: [PATCH 2/3] test(setup): add nested-workspace + polyglot-monorepo layouts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the setup-flow matrix with an `SM_LAYOUT` dimension modelling real-world deployments beyond a single project: - workspace (npm, pnpm, yarn, pip, uv): a root + several members, including a deeply-nested member and one with no dependency on the patched package. Exercises `setup`'s workspace handling — npm/yarn write the hook to every member, pnpm only to the root — and the cross-workspace apply on a single root install. npm/pnpm/yarn apply (the dependency hoists / lands in the pnpm store and is patched once); pip (nested requirements) and uv (uv workspace, one shared .venv) are Python gaps. - monorepo: a polyglot repo with an npm workspace alongside python/rust/go/php/ruby/nuget/deno manifests. Confirms `setup` works in a mixed environment — it configures the npm hooks and does not choke on the foreign manifests; a root `npm install` then patches the npm slice. Runs in the npm image; the foreign manifests are present to test setup's robustness, not installed. Wiring: `matrix.json` gains workspace_targets/scenarios and monorepo_targets/scenarios; `run-case.sh` gains layout-aware scaffold / install / multi-target verification; `scripts/setup-matrix.sh` threads a `layout` column (+ `query --layout`); the Rust harness gains `run_workspace_pm` / `run_monorepo`, with `*_workspace` tests on the npm/pypi wrappers and a new `setup_matrix_monorepo.rs`. Real-world finding (and fix in the harness): the install hook's `apply` must run with the package manager's per-script cwd — root for the project, the member dir for each member — so member postinstalls find no manifest and no-op while the root applies. The driver therefore does NOT pin SOCKET_CWD; pinning it to the root makes every member apply target the root manifest and fail mid-install with "no packages found on disk", breaking `npm install` in a workspace. Verified in Docker (socket-patch 3.3.0): npm/pnpm/yarn workspace and the monorepo apply (pass); pip/uv workspace are known_gap; single-project cases unchanged. 92 cases total; 0 regressions. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tests/setup_matrix_common/mod.rs | 70 ++++-- .../tests/setup_matrix_monorepo.rs | 20 ++ .../tests/setup_matrix_npm.rs | 22 ++ .../tests/setup_matrix_pypi.rs | 15 ++ scripts/setup-matrix.sh | 59 +++-- tests/setup_matrix/README.md | 51 ++++- tests/setup_matrix/matrix.json | 91 ++++++++ tests/setup_matrix/run-case.sh | 215 +++++++++++++++--- 8 files changed, 467 insertions(+), 76 deletions(-) create mode 100644 crates/socket-patch-cli/tests/setup_matrix_monorepo.rs diff --git a/crates/socket-patch-cli/tests/setup_matrix_common/mod.rs b/crates/socket-patch-cli/tests/setup_matrix_common/mod.rs index 4532ab3d..d2fe02c1 100644 --- a/crates/socket-patch-cli/tests/setup_matrix_common/mod.rs +++ b/crates/socket-patch-cli/tests/setup_matrix_common/mod.rs @@ -94,6 +94,7 @@ struct Case { apply_ecosystems: String, marker: String, alt_marker: String, + layout: String, } impl Case { @@ -121,13 +122,24 @@ impl Case { ("SM_APPLY_ECOSYSTEMS".into(), self.apply_ecosystems.clone()), ("SM_MARKER".into(), self.marker.clone()), ("SM_ALT_MARKER".into(), self.alt_marker.clone()), + ("SM_LAYOUT".into(), self.layout.clone()), ] } } -/// Load every case for a given (ecosystem, pm) by crossing that target -/// with all scenarios in the spec. -fn load_cases(ecosystem: &str, pm: &str) -> Vec { +/// Load every case for a given (ecosystem, pm) by crossing the matching +/// target in `targets_key` with every scenario in `scenarios_key`, +/// tagging each with `layout`. `targets_key`/`scenarios_key` select the +/// spec section: ("targets","scenarios") for single projects, +/// ("workspace_targets","workspace_scenarios") for nested workspaces, +/// ("monorepo_targets","monorepo_scenarios") for the polyglot monorepo. +fn load_section( + targets_key: &str, + scenarios_key: &str, + layout: &str, + ecosystem: &str, + pm: &str, +) -> Vec { let text = std::fs::read_to_string(matrix_path()) .unwrap_or_else(|e| panic!("read matrix.json: {e}")); let spec: serde_json::Value = @@ -135,15 +147,15 @@ fn load_cases(ecosystem: &str, pm: &str) -> Vec { let marker = spec["marker"].as_str().unwrap_or("").to_string(); let alt_marker = spec["alt_marker"].as_str().unwrap_or("").to_string(); - let target = spec["targets"] + let target = spec[targets_key] .as_array() - .expect("targets array") + .unwrap_or_else(|| panic!("{targets_key} array missing")) .iter() .find(|t| t["ecosystem"] == ecosystem && t["pm"] == pm) - .unwrap_or_else(|| panic!("no target for {ecosystem}/{pm} in matrix.json")); + .unwrap_or_else(|| panic!("no {targets_key} entry for {ecosystem}/{pm}")); let mut cases = Vec::new(); - for s in spec["scenarios"].as_array().expect("scenarios array") { + for s in spec[scenarios_key].as_array().expect("scenarios array") { let scenario = s["id"].as_str().unwrap().to_string(); cases.push(Case { id: format!("{ecosystem}/{pm}/{scenario}"), @@ -162,6 +174,7 @@ fn load_cases(ecosystem: &str, pm: &str) -> Vec { apply_ecosystems: target["apply_ecosystems"].as_str().unwrap().to_string(), marker: marker.clone(), alt_marker: alt_marker.clone(), + layout: layout.to_string(), }); } cases @@ -222,22 +235,44 @@ fn run_case(case: &Case) -> RunResult { } } -/// Run every scenario for one (ecosystem, pm) and assert each meets the -/// ASPIRATIONAL expectation. Soft-skips when Docker / the ecosystem -/// image is unavailable (container mode) — matching the `docker_e2e_*` -/// convention where Rust integration tests have no native "skipped". +/// Run the single-project scenarios for one (ecosystem, pm). pub fn run_pm(ecosystem: &str, pm: &str) { + run_cases( + &format!("{ecosystem}/{pm}"), + load_section("targets", "scenarios", "single", ecosystem, pm), + ); +} + +/// Run the nested-workspace scenarios for one (ecosystem, pm). +pub fn run_workspace_pm(ecosystem: &str, pm: &str) { + run_cases( + &format!("{ecosystem}/{pm} [workspace]"), + load_section("workspace_targets", "workspace_scenarios", "workspace", ecosystem, pm), + ); +} + +/// Run the polyglot all-ecosystem monorepo scenarios. +pub fn run_monorepo() { + run_cases( + "monorepo", + load_section("monorepo_targets", "monorepo_scenarios", "monorepo", "monorepo", "mono"), + ); +} + +/// Execute a set of cases and assert each meets the ASPIRATIONAL +/// expectation. Soft-skips when Docker / the ecosystem image is +/// unavailable (container mode) — matching the `docker_e2e_*` convention +/// where Rust integration tests have no native "skipped". +fn run_cases(label: &str, cases: Vec) { if !host_mode() && !docker_on_path() { - eprintln!("skip {ecosystem}/{pm}: docker not on PATH (set SOCKET_PATCH_TEST_HOST=1 to run on host)"); + eprintln!("skip {label}: docker not on PATH (set SOCKET_PATCH_TEST_HOST=1 to run on host)"); return; } - - let cases = load_cases(ecosystem, pm); if !host_mode() { if let Some(c) = cases.first() { if !image_present(&c.image) { eprintln!( - "skip {ecosystem}/{pm}: image socket-patch-test-{}:latest not present \ + "skip {label}: image socket-patch-test-{}:latest not present \ (build it: scripts/setup-matrix.sh build --ecosystem {})", c.image, c.image ); @@ -267,12 +302,11 @@ pub fn run_pm(ecosystem: &str, pm: &str) { assert!( failures.is_empty(), - "{}/{}: {} of {} setup-matrix case(s) did not meet the aspirational \ + "{}: {} of {} setup-matrix case(s) did not meet the aspirational \ expectation. BASELINE GAP entries are the experimental TODO list \ (this suite is non-blocking in CI); REGRESSION / LEAK entries are \ real problems:\n{}", - ecosystem, - pm, + label, failures.len(), cases.len(), failures.join("\n") diff --git a/crates/socket-patch-cli/tests/setup_matrix_monorepo.rs b/crates/socket-patch-cli/tests/setup_matrix_monorepo.rs new file mode 100644 index 00000000..f62cb508 --- /dev/null +++ b/crates/socket-patch-cli/tests/setup_matrix_monorepo.rs @@ -0,0 +1,20 @@ +//! setup-matrix: polyglot all-ecosystem monorepo. +//! +//! A single repo containing an npm workspace alongside +//! python/rust/go/php/ruby/nuget/deno manifests. Confirms `socket-patch +//! setup` works in this mixed environment — it must configure the npm +//! hooks and NOT choke on the foreign manifests; a root `npm install` +//! then applies the patch to the npm slice. Runs in the npm image (the +//! only one with the npm toolchain); the foreign manifests are present +//! to test setup's robustness, not installed. +//! +//! Run: `cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_monorepo` +#![cfg(feature = "setup-e2e")] + +#[path = "setup_matrix_common/mod.rs"] +mod smc; + +#[test] +fn monorepo() { + smc::run_monorepo(); +} diff --git a/crates/socket-patch-cli/tests/setup_matrix_npm.rs b/crates/socket-patch-cli/tests/setup_matrix_npm.rs index ac4bd3c0..eac266e6 100644 --- a/crates/socket-patch-cli/tests/setup_matrix_npm.rs +++ b/crates/socket-patch-cli/tests/setup_matrix_npm.rs @@ -31,3 +31,25 @@ fn pnpm() { fn bun() { smc::run_pm("npm", "bun"); } + +// ── Nested-workspace layouts ────────────────────────────────────────── +// A root + several members (incl. a deeply-nested one and a member with +// no dependency on the patched package). Exercises `setup`'s workspace +// handling (npm/yarn write the hook to every member; pnpm only to the +// root) plus the cross-workspace apply on the root install. These should +// PASS — they're real regression guards, not gap documentation. + +#[test] +fn npm_workspace() { + smc::run_workspace_pm("npm", "npm"); +} + +#[test] +fn pnpm_workspace() { + smc::run_workspace_pm("npm", "pnpm"); +} + +#[test] +fn yarn_workspace() { + smc::run_workspace_pm("npm", "yarn"); +} diff --git a/crates/socket-patch-cli/tests/setup_matrix_pypi.rs b/crates/socket-patch-cli/tests/setup_matrix_pypi.rs index e11a9210..b1907433 100644 --- a/crates/socket-patch-cli/tests/setup_matrix_pypi.rs +++ b/crates/socket-patch-cli/tests/setup_matrix_pypi.rs @@ -35,3 +35,18 @@ fn pdm() { fn hatch() { smc::run_pm("pypi", "hatch"); } + +// ── Nested-workspace layouts (EXPECTED BASELINE GAP) ────────────────── +// uv workspace (root + members, one shared .venv) and a pip +// nested-requirements monorepo. Python has no post-install hook, so +// these don't apply today — but the install itself must succeed. + +#[test] +fn pip_workspace() { + smc::run_workspace_pm("pypi", "pip"); +} + +#[test] +fn uv_workspace() { + smc::run_workspace_pm("pypi", "uv"); +} diff --git a/scripts/setup-matrix.sh b/scripts/setup-matrix.sh index 67faa4ba..dbec628d 100755 --- a/scripts/setup-matrix.sh +++ b/scripts/setup-matrix.sh @@ -47,23 +47,29 @@ usage() { sed -n '2,40p' "$0" | sed 's/^# \{0,1\}//'; } need jq [ -f "$MATRIX" ] || die "matrix spec not found: $MATRIX" -# Emit one TSV row per case (target x scenario), honoring filters. +# Emit one TSV row per case, honoring filters. Covers all three layouts: +# single (targets x scenarios), workspace (workspace_targets x +# workspace_scenarios) and monorepo (monorepo_targets x monorepo_scenarios). # Columns: id eco pm image hook_family baseline_supported package version # purl manifest_key apply_ecosystems scenario patchset run_setup -# expect_applied +# expect_applied layout cases_tsv() { # $1=eco-filter ("" = all) $2=pm-filter $3=scenario-filter jq -r --arg eco "${1:-}" --arg pm "${2:-}" --arg scn "${3:-}" ' - .marker as $m | .alt_marker as $am | - .targets[] as $t | .scenarios[] as $s | - select($eco == "" or $t.ecosystem == $eco) | - select($pm == "" or $t.pm == $pm) | - select($scn == "" or $s.id == $scn) | - [ ($t.ecosystem + "/" + $t.pm + "/" + $s.id), - $t.ecosystem, $t.pm, $t.image, $t.hook_family, - ($t.baseline_supported|tostring), - $t.package, $t.version, $t.purl, $t.manifest_key, $t.apply_ecosystems, - $s.id, $s.patchset, ($s.run_setup|tostring), ($s.expect_applied|tostring) - ] | @tsv + def rows($targets; $scenarios; $layout): + $targets[] as $t | $scenarios[] as $s + | select($eco == "" or $t.ecosystem == $eco) + | select($pm == "" or $t.pm == $pm) + | select($scn == "" or $s.id == $scn) + | [ ($t.ecosystem + "/" + $t.pm + "/" + $s.id), + $t.ecosystem, $t.pm, $t.image, ($t.hook_family // ""), + ($t.baseline_supported|tostring), + $t.package, $t.version, $t.purl, $t.manifest_key, $t.apply_ecosystems, + $s.id, $s.patchset, ($s.run_setup|tostring), ($s.expect_applied|tostring), + $layout ] + | @tsv; + rows(.targets; .scenarios; "single"), + rows((.workspace_targets // []); (.workspace_scenarios // []); "workspace"), + rows((.monorepo_targets // []); (.monorepo_scenarios // []); "monorepo") ' "$MATRIX" } @@ -101,9 +107,9 @@ cmd_list() { scenario:$s.id, image:$t.image, hook_family:$t.hook_family, baseline_supported:$t.baseline_supported, expect_applied:$s.expect_applied } ]' "$MATRIX" else - printf '%-44s %-9s %-8s %-22s %s\n' ID ECO PM SCENARIO EXPECT - cases_tsv "" "" "" | while IFS=$'\t' read -r id eco pm image hook bsup pkg ver purl key aeco scn pset rsetup expect; do - printf '%-44s %-9s %-8s %-22s %s\n' "$id" "$eco" "$pm" "$scn" "$expect" + printf '%-46s %-9s %-8s %-11s %-22s %s\n' ID ECO PM LAYOUT SCENARIO EXPECT + cases_tsv "" "" "" | while IFS=$'\t' read -r id eco pm image hook bsup pkg ver purl key aeco scn pset rsetup expect layout; do + printf '%-46s %-9s %-8s %-11s %-22s %s\n' "$id" "$eco" "$pm" "$layout" "$scn" "$expect" done fi } @@ -142,14 +148,15 @@ cmd_run() { fi local total=0 - while IFS=$'\t' read -r id eco_ pm_ image hook bsup pkg ver purl key aeco scn_ pset rsetup expect; do + while IFS=$'\t' read -r id eco_ pm_ image hook bsup pkg ver purl key aeco scn_ pset rsetup expect layout; do [ -z "$id" ] && continue total=$((total+1)) - echo ">> [$total] $id" >&2 + echo ">> [$total] $id (layout=$layout)" >&2 # Common SM_* env for the driver. local -a base_env=( "SM_ID=$id" "SM_ECOSYSTEM=$eco_" "SM_PM=$pm_" "SM_SCENARIO=$scn_" + "SM_LAYOUT=$layout" "SM_PATCHSET=$pset" "SM_RUN_SETUP=$([ "$rsetup" = true ] && echo 1 || echo 0)" "SM_EXPECT_APPLIED=$([ "$expect" = true ] && echo 1 || echo 0)" "SM_PACKAGE=$pkg" "SM_VERSION=$ver" "SM_PURL=$purl" @@ -182,7 +189,7 @@ cmd_run() { if [ "$expect" = true ] && [ "$bsup" = true ]; then bl=true; fi if [ -n "$result" ] && printf '%s' "$result" | jq -e . >/dev/null 2>&1; then - printf '%s\n' "$result" | jq -c --argjson bl "$bl" --arg img "$image" --arg hk "$hook" ' + printf '%s\n' "$result" | jq -c --argjson bl "$bl" --arg img "$image" --arg hk "$hook" --arg lay "$layout" ' . as $r | ($r.actual_applied == $r.expect_applied) as $ideal | ($r.actual_applied == $bl) as $base | @@ -190,15 +197,15 @@ cmd_run() { elif $ideal and ($base|not) then "progress" elif ($ideal|not) and $base then "known_gap" else "regression" end) as $cls | - $r + {baseline_applied:$bl, classification:$cls, image:$img, hook_family:$hk, driver_rc:'"$rc"'} + $r + {baseline_applied:$bl, classification:$cls, layout:$lay, image:$img, hook_family:$hk, driver_rc:'"$rc"'} ' >> "$jsonl" else # No parseable result — surface as an error case. jq -nc --arg id "$id" --arg eco "$eco_" --arg pm "$pm_" --arg scn "$scn_" \ - --arg pset "$pset" --arg img "$image" --arg hk "$hook" --argjson bl "$bl" ' + --arg pset "$pset" --arg img "$image" --arg hk "$hook" --arg lay "$layout" --argjson bl "$bl" ' { id:$id, ecosystem:$eco, pm:$pm, scenario:$scn, patchset:$pset, expect_applied:null, actual_applied:null, baseline_applied:$bl, - classification:"error", image:$img, hook_family:$hk, driver_rc:'"$rc"', + classification:"error", layout:$lay, image:$img, hook_family:$hk, driver_rc:'"$rc"', notes:"driver produced no parseable result" }' >> "$jsonl" fi done < <(cases_tsv "$eco" "$pm" "$scn") @@ -239,21 +246,23 @@ print_summary() { # $1 = results file # --------------------------------------------------------------------- query / results cmd_query() { - local status="" eco="" pm="" scn="" + local status="" eco="" pm="" scn="" lay="" while [ $# -gt 0 ]; do case "$1" in --status) status="$2"; shift 2;; --ecosystem) eco="$2"; shift 2;; --pm) pm="$2"; shift 2;; --scenario) scn="$2"; shift 2;; + --layout) lay="$2"; shift 2;; *) die "query: unknown arg '$1'";; esac; done [ -f "$LATEST" ] || die "no results yet — run '$0 run' first" - jq --arg st "$status" --arg eco "$eco" --arg pm "$pm" --arg scn "$scn" ' + jq --arg st "$status" --arg eco "$eco" --arg pm "$pm" --arg scn "$scn" --arg lay "$lay" ' [ .cases[] | select($st == "" or .classification == $st) | select($eco == "" or .ecosystem == $eco) | select($pm == "" or .pm == $pm) - | select($scn == "" or .scenario == $scn) ]' "$LATEST" + | select($scn == "" or .scenario == $scn) + | select($lay == "" or .layout == $lay) ]' "$LATEST" } cmd_results() { diff --git a/tests/setup_matrix/README.md b/tests/setup_matrix/README.md index 911503cd..fe670f0d 100644 --- a/tests/setup_matrix/README.md +++ b/tests/setup_matrix/README.md @@ -35,13 +35,46 @@ wrappers). - **Package managers:** npm, yarn, pnpm, bun · pip, uv, poetry, pdm, hatch · cargo · bundler · go · mvn · composer · dotnet · deno. -- **Scenarios:** +- **Scenarios (single-project):** - `baseline_with_setup` — setup + install ⇒ patch applied *(ideal)*. - `no_setup_control` — install only ⇒ NOT applied *(the hook is the cause)*. - `empty_patchset` — empty manifest ⇒ NOT applied. - `wrong_target_patchset` — manifest targets a different package ⇒ NOT applied. - `alt_content_patchset` — a second patch set ⇒ its marker applied *(content tracks the manifest)*. +## Layouts + +The driver's `SM_LAYOUT` selects the project shape (each layout has its +own `*_targets` / `*_scenarios` sections in `matrix.json`): + +- **`single`** *(default)* — one project, one dependency. The 16-PM grid above. +- **`workspace`** — a **nested workspace/monorepo**: a root + several + members (incl. a deeply-nested one and a member that does *not* use the + patched package). Models real-world monorepo deployments and exercises + `setup`'s workspace handling — npm/yarn write the hook to **every** + member, pnpm only to the **root** — plus the cross-workspace apply on a + single root install. Covered PMs: **npm, pnpm, yarn** (apply; the + dependency hoists / lands in the pnpm store and is patched once) and + **pip** (nested `requirements.txt` files) + **uv** (uv workspace, one + shared `.venv`) as Python gaps. Scenarios: `workspace_with_setup`, + `workspace_no_setup`. +- **`monorepo`** — a **polyglot all-ecosystem repo**: an npm workspace + alongside python/rust/go/php/ruby/nuget/deno manifests. Confirms `setup` + works in a mixed environment — it must configure the npm hooks and + **not choke** on the foreign manifests; a root `npm install` then + patches the npm slice. Runs in the npm image (the only one with the npm + toolchain), so the foreign manifests are present to test setup's + robustness, not installed. Scenarios: `monorepo_with_setup`, + `monorepo_no_setup`. + +> Real-world wiring note surfaced by the workspace layout: the install +> hook's `apply` must run with the package manager's per-script cwd (root +> for the project, the member dir for each member) — so member +> postinstalls find no manifest and no-op while the root applies. Forcing +> a single cwd makes every member target the root manifest and fail +> mid-install with "no packages found on disk". The driver therefore does +> **not** pin `SOCKET_CWD`. + ## Result classification Each case's `actual` is compared against both the aspirational `expect` @@ -76,9 +109,14 @@ scripts/setup-matrix.sh run --ecosystem npm scripts/setup-matrix.sh run --ecosystem pypi --pm uv scripts/setup-matrix.sh run --scenario no_setup_control +# Run the nested-workspace and polyglot-monorepo cases. +scripts/setup-matrix.sh run --scenario workspace_with_setup +scripts/setup-matrix.sh run --scenario monorepo_with_setup + # Query the last results (agent-friendly JSON). scripts/setup-matrix.sh query --status known_gap scripts/setup-matrix.sh query --status regression +scripts/setup-matrix.sh query --layout workspace scripts/setup-matrix.sh list --json # Host mode (no Docker; needs the toolchains + a built binary on PATH). @@ -95,9 +133,11 @@ SOCKET_PATCH_TEST_HOST=1 cargo test -p socket-patch-cli --features setup-e2e --t ## Files -- `matrix.json` — declarative case list (targets × scenarios) + markers. -- `run-case.sh` — self-contained flow driver (one case → JSON result); - generates the runner shims inline so it can be piped into a container. +- `matrix.json` — declarative case list: `targets`×`scenarios` (single), + `workspace_targets`×`workspace_scenarios`, `monorepo_targets`×`monorepo_scenarios`, + markers. +- `run-case.sh` — self-contained flow driver (one case → JSON result), + layout-aware (`SM_LAYOUT=single|workspace|monorepo`); generates the + runner shims inline so it can be piped into a container. - `shims/{npx,pnpm}` — reference copies of the PATH shims that route `npx`/`pnpm dlx @socketsecurity/socket-patch` to the locally-built binary (so the hook runs the binary under test, not a registry fetch). @@ -105,7 +145,8 @@ SOCKET_PATCH_TEST_HOST=1 cargo test -p socket-patch-cli --features setup-e2e --t - `../docker/Dockerfile.{npm,pypi,…}` — the per-ecosystem images (npm/pypi extended with the extra package managers). - `../../crates/socket-patch-cli/tests/setup_matrix_.rs` — thin Rust - wrappers around the same driver. + wrappers around the same driver (incl. `setup_matrix_monorepo.rs`; the + npm/pypi wrappers add `*_workspace` tests). ## Adding a package manager / ecosystem diff --git a/tests/setup_matrix/matrix.json b/tests/setup_matrix/matrix.json index 65a1a177..5159cc63 100644 --- a/tests/setup_matrix/matrix.json +++ b/tests/setup_matrix/matrix.json @@ -165,5 +165,96 @@ "package": "minimist", "version": "1.2.2", "purl": "pkg:npm/minimist@1.2.2", "manifest_key": "package/index.js", "apply_ecosystems": "npm" } + ], + + "_workspace_comment": [ + "Nested-workspace layouts (run-case.sh SM_LAYOUT=workspace): a root +", + "several workspace members (incl. a deeply-nested one and a member that", + "does NOT use the patched package). Models real monorepos and exercises", + "`setup`'s workspace handling — npm/yarn write the hook to every member,", + "pnpm only to the root — plus the cross-workspace apply on the root", + "install. npm/yarn/pnpm should apply (baseline_supported true); Python", + "workspaces (uv workspace, pip nested-requirements) are gaps." + ], + "workspace_scenarios": [ + { + "id": "workspace_with_setup", + "run_setup": true, + "patchset": "primary", + "expect_applied": true, + "description": "Nested workspace: setup at root, then a root-level install must apply the patch to the (hoisted/store-linked) dependency used across members." + }, + { + "id": "workspace_no_setup", + "run_setup": false, + "patchset": "primary", + "expect_applied": false, + "description": "Negative control: workspace install without setup must NOT apply." + } + ], + "workspace_targets": [ + { + "ecosystem": "npm", "pm": "npm", "image": "npm", "hook_family": "npm", + "baseline_supported": true, + "package": "minimist", "version": "1.2.2", "purl": "pkg:npm/minimist@1.2.2", + "manifest_key": "package/index.js", "apply_ecosystems": "npm" + }, + { + "ecosystem": "npm", "pm": "pnpm", "image": "npm", "hook_family": "pnpm", + "baseline_supported": true, + "package": "minimist", "version": "1.2.2", "purl": "pkg:npm/minimist@1.2.2", + "manifest_key": "package/index.js", "apply_ecosystems": "npm" + }, + { + "ecosystem": "npm", "pm": "yarn", "image": "npm", "hook_family": "npm", + "baseline_supported": true, + "package": "minimist", "version": "1.2.2", "purl": "pkg:npm/minimist@1.2.2", + "manifest_key": "package/index.js", "apply_ecosystems": "npm" + }, + { + "ecosystem": "pypi", "pm": "pip", "image": "pypi", "hook_family": "none", + "baseline_supported": false, + "package": "six", "version": "1.16.0", "purl": "pkg:pypi/six@1.16.0", + "manifest_key": "six.py", "apply_ecosystems": "pypi" + }, + { + "ecosystem": "pypi", "pm": "uv", "image": "pypi", "hook_family": "none", + "baseline_supported": false, + "package": "six", "version": "1.16.0", "purl": "pkg:pypi/six@1.16.0", + "manifest_key": "six.py", "apply_ecosystems": "pypi" + } + ], + + "_monorepo_comment": [ + "Polyglot monorepo (SM_LAYOUT=monorepo): an npm workspace alongside", + "python/rust/go/php/ruby/nuget/deno manifests. Confirms `setup` works in", + "a mixed environment — it must configure the npm hooks and NOT choke on", + "the foreign manifests; a root `npm install` then patches the npm slice.", + "Runs in the npm image (the only one with the npm toolchain); the foreign", + "manifests are present to test setup's robustness, not installed." + ], + "monorepo_scenarios": [ + { + "id": "monorepo_with_setup", + "run_setup": true, + "patchset": "primary", + "expect_applied": true, + "description": "All ecosystems present: setup at root, then npm install applies the patch to the npm workspace dependency; setup must not error on the foreign manifests." + }, + { + "id": "monorepo_no_setup", + "run_setup": false, + "patchset": "primary", + "expect_applied": false, + "description": "Negative control: polyglot monorepo install without setup must NOT apply." + } + ], + "monorepo_targets": [ + { + "ecosystem": "monorepo", "pm": "mono", "image": "npm", "hook_family": "npm", + "baseline_supported": true, + "package": "minimist", "version": "1.2.2", "purl": "pkg:npm/minimist@1.2.2", + "manifest_key": "package/index.js", "apply_ecosystems": "npm" + } ] } diff --git a/tests/setup_matrix/run-case.sh b/tests/setup_matrix/run-case.sh index 668db381..c003596a 100755 --- a/tests/setup_matrix/run-case.sh +++ b/tests/setup_matrix/run-case.sh @@ -60,6 +60,7 @@ SM_MANIFEST_KEY="${SM_MANIFEST_KEY:-package/index.js}" SM_APPLY_ECOSYSTEMS="${SM_APPLY_ECOSYSTEMS:-npm}" SM_MARKER="${SM_MARKER:-SOCKET-PATCH-SETUP-MATRIX-MARKER}" SM_ALT_MARKER="${SM_ALT_MARKER:-SOCKET-PATCH-SETUP-MATRIX-ALT-MARKER}" +SM_LAYOUT="${SM_LAYOUT:-single}" ZEROHASH="0000000000000000000000000000000000000000000000000000000000000000" UUID="aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" @@ -303,8 +304,144 @@ resolve_target() { esac } +# --- workspace scaffold (root + nested members) ---------------------- +# Models a real monorepo where multiple (incl. deeply-nested) workspace +# members depend on the package being patched, plus a member that does +# NOT — so `setup`'s workspace handling (npm: every member; pnpm: root +# only) and the root install's cross-workspace apply are both exercised. +ws_member_js() { # $1=dir $2=name (declares the dep) + mkdir -p "$1" + cat > "$1/package.json" < package.json <<'EOF' +{ "name": "sm-root", "version": "0.0.0", "private": true, + "workspaces": ["packages/*", "packages/group/*"] } +EOF + ws_member_js packages/app "@sm/app" + ws_member_js packages/lib "@sm/lib" + ws_member_js packages/group/nested "@sm/nested" + mkdir -p packages/util # member with NO dependency on the patched pkg + printf '{ "name": "@sm/util", "version": "0.0.0", "private": true }\n' > packages/util/package.json ;; + pnpm) + printf '{ "name": "sm-root", "version": "0.0.0", "private": true }\n' > package.json + cat > pnpm-workspace.yaml <<'EOF' +packages: + - 'packages/*' + - 'packages/group/*' +EOF + ws_member_js packages/app "@sm/app" + ws_member_js packages/lib "@sm/lib" + ws_member_js packages/group/nested "@sm/nested" + mkdir -p packages/util + printf '{ "name": "@sm/util", "version": "0.0.0", "private": true }\n' > packages/util/package.json ;; + uv) + # uv workspace: virtual root + members; the shared dep is installed + # into one root .venv by `uv sync`. + cat > pyproject.toml < "packages/$m/pyproject.toml" < packages/app/requirements.txt + echo "$SM_PACKAGE==$SM_VERSION" > packages/lib/requirements.txt + printf -- '-r packages/app/requirements.txt\n-r packages/lib/requirements.txt\n' > requirements.txt ;; + esac +} + +run_install_workspace() { + case "$SM_PM" in + npm) npm install --silent --no-audit --no-fund ;; + yarn) yarn install --silent ;; + pnpm) pnpm install --no-frozen-lockfile ;; + uv) uv sync ;; + pip) python3 -m venv venv && ./venv/bin/pip install --disable-pip-version-check --quiet --no-cache-dir -r requirements.txt ;; + esac +} + +# --- all-ecosystem monorepo scaffold --------------------------------- +# A polyglot repo: an npm workspace (the slice `setup` supports AND the +# npm image can install) alongside python/rust/go/php/ruby/nuget/deno +# manifests. The point is to confirm `setup` works in this environment — +# it must configure the npm hooks and NOT choke on the foreign manifests. +scaffold_monorepo() { + cat > package.json <<'EOF' +{ "name": "sm-monorepo", "version": "0.0.0", "private": true, + "workspaces": ["packages/js-*"] } +EOF + ws_member_js packages/js-app "@mono/js-app" + ws_member_js packages/js-nested "@mono/js-nested" + mkdir -p packages/py-svc + cat > packages/py-svc/pyproject.toml <<'EOF' +[project] +name = "py-svc" +version = "0.0.0" +requires-python = ">=3.9" +dependencies = ["six==1.16.0"] +EOF + printf 'six==1.16.0\n' > packages/py-svc/requirements.txt + mkdir -p packages/rust-lib/src + printf '[package]\nname = "rust-lib"\nversion = "0.0.0"\nedition = "2021"\n\n[dependencies]\ncfg-if = "=1.0.0"\n' > packages/rust-lib/Cargo.toml + printf '// lib\n' > packages/rust-lib/src/lib.rs + mkdir -p packages/go-mod && printf 'module mono/go\n\ngo 1.21\n' > packages/go-mod/go.mod + mkdir -p packages/php-web && printf '{ "name": "mono/php", "require": { "monolog/monolog": "3.5.0" } }\n' > packages/php-web/composer.json + mkdir -p packages/ruby-gem && printf "source 'https://rubygems.org'\ngem 'colorize', '1.1.0'\n" > packages/ruby-gem/Gemfile + mkdir -p packages/deno-app && printf '{ "name": "mono/deno", "version": "0.0.0" }\n' > packages/deno-app/deno.json + mkdir -p packages/nuget-app && printf '\n' > packages/nuget-app/app.csproj +} + +run_install_monorepo() { + npm install --silent --no-audit --no-fund +} + +# --- resolve candidate on-disk file(s) for verification -------------- +# For single layout: one path. For workspace/monorepo: search the tree +# (hoisted root node_modules, pnpm store, member dirs, shared venv). +resolve_targets() { + local rel="${SM_MANIFEST_KEY#package/}" + local base; base="$(basename "$rel")" + if [ "$SM_LAYOUT" = single ]; then + resolve_target + return + fi + case "$SM_ECOSYSTEM" in + npm|deno|monorepo) find "$PWD" -path "*/node_modules/$SM_PACKAGE/$rel" 2>/dev/null ;; + pypi) find "$PWD" -name "$base" 2>/dev/null ;; + *) resolve_target ;; + esac +} + # ============================ main ==================================== -log "binary: $SP_BIN ($("$SP_BIN" --version 2>/dev/null || echo '??'))" +log "binary: $SP_BIN ($("$SP_BIN" --version 2>/dev/null || echo '??')) layout=$SM_LAYOUT" WORKDIR="${SM_WORKDIR:-$(mktemp -d)}" PROJ="$WORKDIR/proj" @@ -313,19 +450,23 @@ cd "$PROJ" || { emit_result false null null null "" fail; exit 0; } note "proj=$PROJ" # 0. dependencies + committed patch set -scaffold_project +case "$SM_LAYOUT" in + workspace) scaffold_workspace ;; + monorepo) scaffold_monorepo ;; + *) scaffold_project ;; +esac build_fixture -# npm-family (incl. deno-via-npm) need the runner shim so the hook's -# `npx`/`pnpm dlx @socketsecurity/socket-patch` resolves to $SP_BIN. -case "$SM_PM" in - npm|yarn|pnpm|bun|deno) - SHIM_DIR="$PROJ/.sp-shims" - write_shims "$SHIM_DIR" - export SETUP_MATRIX_SHIM_DIR="$SHIM_DIR" - export PATH="$SHIM_DIR:$PATH" - log "shims installed at $SHIM_DIR (PATH prepended)" ;; -esac +# npm-family (incl. deno-via-npm and the monorepo's npm slice) need the +# runner shim so the hook's `npx`/`pnpm dlx @socketsecurity/socket-patch` +# resolves to $SP_BIN instead of the npm registry. +if [[ "$SM_PM" =~ ^(npm|yarn|pnpm|bun|deno)$ ]] || [ "$SM_LAYOUT" = monorepo ]; then + SHIM_DIR="$PROJ/.sp-shims" + write_shims "$SHIM_DIR" + export SETUP_MATRIX_SHIM_DIR="$SHIM_DIR" + export PATH="$SHIM_DIR:$PATH" + log "shims installed at $SHIM_DIR (PATH prepended)" +fi # Hermetic apply env inherited by the install hook's `socket-patch apply`. # NOTE: SOCKET_OFFLINE/SOCKET_FORCE must be "true"/"false" — the apply @@ -335,7 +476,14 @@ esac # "1". export SOCKET_OFFLINE=true SOCKET_FORCE=true SOCKET_API_TOKEN=fake SOCKET_ORG_SLUG=test-org export SOCKET_TELEMETRY_DISABLED=1 SOCKET_EXPERIMENTAL_MAVEN=1 SOCKET_EXPERIMENTAL_NUGET=1 -export SOCKET_CWD="$PROJ" +# NOTE: deliberately do NOT export SOCKET_CWD. The install hook's apply +# must run with whatever cwd the package manager sets for the lifecycle +# script — the project root for a single project, and the *member* dir +# for each workspace member. In a workspace, member postinstalls thus +# find no manifest in their own dir and no-op (exit 0), while the root +# postinstall (manifest present) applies. Forcing SOCKET_CWD=root would +# make every member apply target the root manifest and fail with "no +# packages found on disk" mid-install, breaking `npm install`. # 1. setup (configures hooks; no-op where there is no package.json) SETUP_EXIT="null" @@ -347,25 +495,36 @@ if [ "$SM_RUN_SETUP" = 1 ]; then fi # 2. native install (this is where a configured hook fires) -log "running install for pm=$SM_PM" -run_install; INSTALL_EXIT=$? +log "running install for pm=$SM_PM (layout=$SM_LAYOUT)" +case "$SM_LAYOUT" in + workspace) run_install_workspace ;; + monorepo) run_install_monorepo ;; + *) run_install ;; +esac +INSTALL_EXIT=$? log "install exit=$INSTALL_EXIT" -# 3. verify -TARGET="$(resolve_target || true)" -log "resolved target: ${TARGET:-}" +# 3. verify — applied if ANY discovered copy of the patched file carries +# the expected marker (covers hoisting, the pnpm store, member dirs and +# the shared venv in workspace/monorepo layouts). +check_marker="$SM_MARKER" +[ "$SM_PATCHSET" = alt ] && check_marker="$SM_ALT_MARKER" APPLIED=false PRIMARY_PRESENT=null -if [ -n "$TARGET" ] && [ -f "$TARGET" ]; then - # The marker we expect depends on the patch set. - check_marker="$SM_MARKER" - [ "$SM_PATCHSET" = alt ] && check_marker="$SM_ALT_MARKER" - if grep -q "$check_marker" "$TARGET" 2>/dev/null; then APPLIED=true; fi - if grep -q "$SM_MARKER" "$TARGET" 2>/dev/null; then PRIMARY_PRESENT=true; else PRIMARY_PRESENT=false; fi - log "marker '$check_marker' present: $APPLIED" -else - note "target file not found" -fi +TARGET="" +n_found=0 +while IFS= read -r cand; do + [ -n "$cand" ] && [ -f "$cand" ] || continue + n_found=$((n_found + 1)) + [ -z "$TARGET" ] && TARGET="$cand" + if grep -q "$check_marker" "$cand" 2>/dev/null; then APPLIED=true; TARGET="$cand"; fi + if grep -q "$SM_MARKER" "$cand" 2>/dev/null; then PRIMARY_PRESENT=true; fi +done < <(resolve_targets) +[ "$PRIMARY_PRESENT" = null ] && [ "$n_found" -gt 0 ] && PRIMARY_PRESENT=false +note "candidate files found: $n_found" +log "resolved target: ${TARGET:-} (candidates=$n_found)" +[ "$n_found" -eq 0 ] && note "target file not found" +log "marker '$check_marker' present: $APPLIED" # Driver-level status: did actual match the aspirational expectation? want=$([ "$SM_EXPECT_APPLIED" = 1 ] && echo true || echo false) From fe7035c8fbca0fae1767d338938966854ce67d83 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Tue, 2 Jun 2026 13:24:16 -0400 Subject: [PATCH 3/3] test(setup): add patch-missing ablation controls across all layouts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a `patch_missing` ablation (new `patchset: none` — no `.socket/` fixture committed) to the single, workspace and monorepo scenario sets, complementing the existing setup-not-run controls. Together they are the controls that confirm `setup` is correct: each is identical to the corresponding `*_with_setup` case except for the single removed factor (the setup step, or the committed patch), and each must run UNPATCHED. So every "it applies" case is now flanked by both ablations, e.g. for single npm: baseline_with_setup -> applied (patch + setup) no_setup_control -> unpatched (setup ablated) patch_missing -> unpatched (patch ablated) `run-case.sh` skips the fixture entirely for `patchset: none` (so the hook's apply finds no manifest and no-ops — distinct from `empty`, where the manifest exists but lists zero patches). No orchestrator/Rust changes needed; the scenarios are data-driven and picked up automatically. Matrix grows to 114 cases. Verified in Docker (3.3.0): all 22 patch_missing cases pass (run unpatched) — single 16/16, workspace 5/5, monorepo 1/1; with_setup cases still apply; 0 regressions. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/setup_matrix/README.md | 15 +++++++++++---- tests/setup_matrix/matrix.json | 25 +++++++++++++++++++++++-- tests/setup_matrix/run-case.sh | 8 ++++++++ 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/tests/setup_matrix/README.md b/tests/setup_matrix/README.md index fe670f0d..1a66ce05 100644 --- a/tests/setup_matrix/README.md +++ b/tests/setup_matrix/README.md @@ -37,11 +37,18 @@ wrappers). hatch · cargo · bundler · go · mvn · composer · dotnet · deno. - **Scenarios (single-project):** - `baseline_with_setup` — setup + install ⇒ patch applied *(ideal)*. - - `no_setup_control` — install only ⇒ NOT applied *(the hook is the cause)*. - - `empty_patchset` — empty manifest ⇒ NOT applied. + - `no_setup_control` — **ablation (setup not run)**: install only ⇒ NOT applied *(the hook is the cause)*. + - `patch_missing` — **ablation (patch missing)**: setup runs and the hook fires, but no `.socket/` patch set is committed ⇒ runs UNPATCHED *(the committed patch is the cause)*. + - `empty_patchset` — manifest present but with zero patches ⇒ NOT applied. - `wrong_target_patchset` — manifest targets a different package ⇒ NOT applied. - `alt_content_patchset` — a second patch set ⇒ its marker applied *(content tracks the manifest)*. + The two **ablations** are the controls that confirm `setup` is correct: + each is identical to `baseline_with_setup` except for the single removed + factor (the setup step, or the committed patch), and each must run + unpatched. The workspace and monorepo layouts carry the same pair + (`*_no_setup`, `*_patch_missing`). + ## Layouts The driver's `SM_LAYOUT` selects the project shape (each layout has its @@ -57,7 +64,7 @@ own `*_targets` / `*_scenarios` sections in `matrix.json`): dependency hoists / lands in the pnpm store and is patched once) and **pip** (nested `requirements.txt` files) + **uv** (uv workspace, one shared `.venv`) as Python gaps. Scenarios: `workspace_with_setup`, - `workspace_no_setup`. + `workspace_no_setup`, `workspace_patch_missing`. - **`monorepo`** — a **polyglot all-ecosystem repo**: an npm workspace alongside python/rust/go/php/ruby/nuget/deno manifests. Confirms `setup` works in a mixed environment — it must configure the npm hooks and @@ -65,7 +72,7 @@ own `*_targets` / `*_scenarios` sections in `matrix.json`): patches the npm slice. Runs in the npm image (the only one with the npm toolchain), so the foreign manifests are present to test setup's robustness, not installed. Scenarios: `monorepo_with_setup`, - `monorepo_no_setup`. + `monorepo_no_setup`, `monorepo_patch_missing`. > Real-world wiring note surfaced by the workspace layout: the install > hook's `apply` must run with the package manager's per-script cwd (root diff --git a/tests/setup_matrix/matrix.json b/tests/setup_matrix/matrix.json index 5159cc63..ed6ccbdc 100644 --- a/tests/setup_matrix/matrix.json +++ b/tests/setup_matrix/matrix.json @@ -55,6 +55,13 @@ "patchset": "alt", "expect_applied": true, "description": "Different patch set: a second fixture whose blob carries the ALT marker. Proves the applied bytes track the active manifest (alt marker present, primary marker absent)." + }, + { + "id": "patch_missing", + "run_setup": true, + "patchset": "none", + "expect_applied": false, + "description": "Ablation: setup runs and the install hook fires, but NO patch set is committed (no .socket/). The install must run UNPATCHED — proving the committed patch is what changes the code, not setup/install alone." } ], @@ -189,7 +196,14 @@ "run_setup": false, "patchset": "primary", "expect_applied": false, - "description": "Negative control: workspace install without setup must NOT apply." + "description": "Ablation (setup not run): workspace install without setup must NOT apply — the hook is the cause." + }, + { + "id": "workspace_patch_missing", + "run_setup": true, + "patchset": "none", + "expect_applied": false, + "description": "Ablation (patch missing): workspace setup + install with NO committed patch set must run unpatched across the workspace." } ], "workspace_targets": [ @@ -246,7 +260,14 @@ "run_setup": false, "patchset": "primary", "expect_applied": false, - "description": "Negative control: polyglot monorepo install without setup must NOT apply." + "description": "Ablation (setup not run): polyglot monorepo install without setup must NOT apply." + }, + { + "id": "monorepo_patch_missing", + "run_setup": true, + "patchset": "none", + "expect_applied": false, + "description": "Ablation (patch missing): polyglot monorepo setup + install with NO committed patch set must run unpatched." } ], "monorepo_targets": [ diff --git a/tests/setup_matrix/run-case.sh b/tests/setup_matrix/run-case.sh index c003596a..328587be 100755 --- a/tests/setup_matrix/run-case.sh +++ b/tests/setup_matrix/run-case.sh @@ -153,6 +153,14 @@ EOF } build_fixture() { + # Ablation: no patch set committed at all (no .socket/). Even with a + # working install hook, apply finds no manifest and no-ops, so the + # install must run unpatched. Distinct from `empty` (manifest present + # but with zero patches). + if [ "$SM_PATCHSET" = none ]; then + note "no patch fixture committed (ablation: patch missing)" + return + fi mkdir -p .socket/blobs case "$SM_PATCHSET" in empty)